Blame for engine/predict_yearly.py

#!/usr/bin/env python3
"""
1-Year Stock Price & Gain Predictor — Full Alpha Vantage Intelligence
=====================================================================
Predicts 12-month (252 trading day) stock returns using:

  - Point-in-time fundamental features fed INTO ML (not just scored separately)
  - Macro time-series features (yield curve, CPI trend, unemployment)
  - 10 years of price history for ML training
  - 3-model LLM debate (optional annotation only — not blended)

Key differences from predict_monthly.py:
  - Target: 252-day excess return vs SPY (alpha label); ML output converted to absolute + SPY baseline for display
  - ML features include fundamentals + macro (point-in-time mapped)
  - Blend weights: Fund Fair Value 55%, Macro 15%, Direct Price ML 30%
  - Wider return ranges (sanity-bounded blends and ML outputs)
  - Walk-forward with ~252-day OOS windows

Data Sources (17 API endpoints — adds 2Y Treasury for yield curve):
  1-16. Same as predict_monthly.py
  17.   TREASURY_YIELD 2Y — for yield curve slope

Usage:
    python3 predict_yearly.py AAPL
    python3 predict_yearly.py MSFT --llm
"""

from llm_compat import claude_create  # SDK-drift shim for Claude calls
from units import (BASIS_POINTS, DECIMAL_FRACTION, PERCENT_NUMBER, RATIO,  # one formatting path
                   format_percent as _fmt_pct, format_pp as _fmt_pp, format_ratio as _fmt_ratio)
import argparse
import io
import json
import re
import os
import time as _time_mod
_PROCESS_T0 = _time_mod.monotonic()   # the run's start (one subprocess per job)
os.environ.setdefault("PYTHONWARNINGS", "ignore")  # suppress joblib/sklearn thread warnings
import warnings
warnings.filterwarnings("ignore")

from collections import Counter
import sys
import time
from datetime import datetime, timedelta
from pathlib import Path

import numpy as np
import pandas as pd
from scipy.optimize import nnls
from scipy.stats import spearmanr
from sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, RandomForestRegressor
from sklearn.linear_model import ElasticNet, RidgeCV
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from xgboost import XGBRegressor

# ─── Import shared infrastructure from predict_monthly ──────────────────────

from predict_monthly import (
    api_call,
    safe_float,
    _rate_limit,
    analyze_fundamentals,       # 32-point scorecard (reused as-is)
    analyze_sentiment,          # sentiment composite
    _call_openai,
    _call_kimi,
    _call_claude,
    _call_gemini,
    _call_gemini_3_pro,
    _call_claude_synthesis,
    _parse_llm_response,
    _sigmoid_prob,
    _get_llm_calibration_monthly,
    _buyback_rate_yoy_from_shares,
    _iv_percentile_60d,
    fetch_intraday_volume_profile,
    _dedup_flags,
    _NEUTRAL_SYSTEM_PROMPT,
    _BEAR_SYSTEM_PROMPT,
    _BULL_SYSTEM_PROMPT,
    _NEUTRAL_USER_TEMPLATE,
    _BEAR_USER_TEMPLATE,
    _BULL_USER_TEMPLATE,
    _BEAR_REBUTTAL_TEMPLATE,
    _BULL_REBUTTAL_TEMPLATE,
    _PM_SYNTHESIS_USER_TEMPLATE,
    CACHE_DIR,
    API_KEY,
    BASE_URL,
    OPENAI_API_KEY,
    GEMINI_API_KEY,
    ANTHROPIC_API_KEY,
    KIMI_API_KEY,
    BOLD,
    DIM,
    GREEN,
    RED,
    YELLOW,
    CYAN,
    MAGENTA,
    BLUE,
    WHITE,
    RESET,
    BG_GREEN,
    BG_RED,
)
# ── 12-month rubric for the advocate personas ───────────────────────────────────────
# The persona system prompts are written for the MONTHLY engine ("STRONG BUY: upside >5% in
# 1 month", "a 2% drawdown is noise"). Reused unchanged on a 12-month horizon every advocate
# trips STRONG BUY/SELL trivially, so debates polarised into +35% vs -28% regardless of the
# ticker. Swap in thresholds consistent with the app's own 12m rating bands
# (BUY >= +10%, HOLD -5..+10, SELL < -5).
import re as _re_rubric
def _yearly_rubric(prompt: str, side: str) -> str:
    if side == "bear":
        block = ("Recommendation guidelines (12-MONTH horizon, from your perspective):\n"
                 "- STRONG SELL: high-conviction 12-month downside worse than -15%, multiple confirming signals\n"
                 "- SELL: expected 12-month downside worse than -5% with reasonable confidence\n"
                 "- HOLD: risk/reward balanced but tilted bearish (-5%..+10%); not worth adding\n"
                 "Calibrate expected_return_pct to a realistic 12-month outcome, not a drawdown-scare number.\n")
    elif side == "bull":
        block = ("Recommendation guidelines (12-MONTH horizon, from your perspective):\n"
                 "- STRONG BUY: high-conviction 12-month upside above +25% with strong alignment\n"
                 "- BUY: expected 12-month upside above +10% with reasonable confidence\n"
                 "- HOLD: upside exists but under +10%, or entry/timing is not yet ideal\n"
                 "Calibrate expected_return_pct to a realistic 12-month outcome, not a best-case blue-sky number.\n")
    else:
        return prompt
    out = _re_rubric.sub(r"Recommendation guidelines[\s\S]*?(?=\nYou MUST respond)", block, prompt, count=1)
    out = out.replace("A missed 10% move is a failure. A 2% drawdown is noise.",
                      "Over 12 months a missed 25% move is a failure; a 5% drawdown is noise.")
    out = out.replace("in 1 month", "over 12 months")
    return out
_BEAR_SYSTEM_PROMPT = _yearly_rubric(_BEAR_SYSTEM_PROMPT, "bear")
_BULL_SYSTEM_PROMPT = _yearly_rubric(_BULL_SYSTEM_PROMPT, "bull")

# ── Readability + canonical-naming rules (yearly/Crucible only) ─────────────────────
# The debate is read by smart non-professionals: audit (2026-08-31) found internal flag
# names leaking into analyst text, four different names for the same valuation number,
# scores cited without scales, and 60-word sentences. These rules fix tone, not substance.
_STYLE_RULES = """
WRITING RULES (strict — these govern style, not your view):
- Write for a smart reader who is NOT a finance professional. Keep sentences under ~25 words;
  split dense claims into two sentences rather than stacking clauses.
- Expand any technical acronym the first time you use it (e.g. "HBM (high-bandwidth AI memory)").
- NEVER quote internal signal or flag names (e.g. CYCLE_PEAK_ANCHOR, VALUE_TRAP_RISK,
  COMPOUNDER_FLOOR). Describe the adjustment in plain English instead
  ("the model normalizes earnings toward mid-cycle levels").
- When citing a score, state its scale: "insider score -0.98 (on a -1 bearish to +1 bullish scale)".
- EVIDENCE RULE: only cite corporate events, trial or regulatory outcomes, lawsuits, deals or
  product results that appear in the signals, headlines or events block provided, or that are
  well-established facts. NEVER invent or invert an outcome (a positive trial is not a "failed"
  one). Model outputs must be labeled as model estimates ("the engine's forecast"), not stated as
  reported facts. If you are unsure whether something happened, say so or leave it out.
- PENDING vs DONE: a deal that is only announced or agreed is PENDING — say "agreed to acquire",
  never describe it as completed or as a current contributor to revenue or margins.
- ARITHMETIC: every product, ratio or percentage you state must be correct to the figures you cite.
  The market composite is a BLEND of several methods; it is never "EPS × one multiple". If you
  want to cite a multiple, cite the forward P/E at the current price and say so.
- ATTRIBUTION: the valuation is Crucible's own application of a published framework. Write "the
  story value" or "the Damodaran-style value" — never "Damodaran's value/DCF/target", which implies
  he produced or endorsed the number.
- The intrinsic-value gap you cite is the STORY VALUE vs price. The mechanical DCF is a cross-check:
  name it as such and never present its gap as "the intrinsic gap" or "the premium to fair value".
- Use ONLY these canonical names for the valuation anchors, exactly:
  "story value" (the Damodaran story-driven intrinsic value), "mechanical DCF" (the formula-based
  cross-check), "market composite" (the multiples + analyst-target pricing blend), and
  "the engine's 12-month forecast" (the quant/ML model). Do not invent synonyms such as
  "intrinsic DCF", "blended fair value" or "pricing-based fair value".
"""
_ARGUMENT_RULES = """
ARGUMENT RULES (how to write the case — a reader must get your thesis WITHOUT reading the signals):
- "reasoning" is your THESIS in 2-3 sentences (max 60 words): the claim, the single strongest
  reason, and what the market is missing or already paying for. It is a conclusion, not a list.
- "key_points" are the 3-5 HIGHLIGHTS that carry the thesis. Write each as a conclusion FIRST,
  then the one or two numbers that prove it, then what it implies for the 12-month return.
  Never paste a signal block or recite model outputs line by line; a point that only repeats
  figures without saying what they mean is not allowed. Do not reuse a number across points.
- Do not narrate what another analyst said unless you are answering it.
- Prefer the decisive over the comprehensive: drop any topic where you have nothing that
  would change the answer.
- RETURN ARITHMETIC: a 12-month return ≈ EPS growth + change in the multiple + dividend yield. If you
  say the multiple stays where it is, your return is roughly the EPS growth — not double it.
- OWNERSHIP AND INSIDER FILINGS (Form 13F, Form 4) record positions and trades, never intent: state
  what was held or traded and, for insiders, whether it is material relative to the company
  ([[INSIDER_MATERIALITY]] of market cap when listed). Never infer conviction, confidence, belief or
  any other motive, and never call investors "smart" or "informed".
- COMPANY GUIDANCE is only what the company's own earnings release states. Describe guidance only by
  citing a [[GUIDANCE_n]] token; when none is listed, company guidance is unavailable — do not
  mention guidance, outlook targets or what management "expects".
- The engine's revenue, EPS and margin forecasts are MODEL estimates: write "the engine forecasts",
  never "guidance", "consensus" or "embedded in consensus".
- LIKE FOR LIKE: compare trailing multiples with trailing multiples and forward with forward; when a
  RESTATEMENT is flagged, compare growth continuing-company vs continuing-company and never call
  a perimeter change (a unit that left) a decline.
- MEMORY CLAIMS: a specific historical figure or event about the company that is NOT in the data
  provided (a past deal's synergy outcome, a regulator's active probe, an integration cost) must be
  written as "from memory:" and kept qualitative — or left out. Never invert an outcome.
- HARD LIMITS: "reasoning" at most 60 words; each key_point at most 45 words. Longer text is
  cut off mid-sentence in the display, so say the important thing first and stop.
"""
# the semantic rules the PM must honour as well (the length and layout rules above are the seats' only)
def _rule_bullets(text, prefixes):
    out, cur = [], None
    for ln in text.splitlines():
        if ln.startswith("- "):
            cur = [ln] if ln.startswith(prefixes) else None
            if cur is not None:
                out.append(cur)
        elif ln.startswith("  ") and cur is not None:
            cur.append(ln)
        else:
            cur = None
    return "\n".join("\n".join(b) for b in out)


_EVIDENCE_RULES = ("\nEVIDENCE RULES (these bind your synthesis exactly as they bind the advocates):\n"
                   + _rule_bullets(_ARGUMENT_RULES, ("- RETURN", "- OWNERSHIP", "- COMPANY GUIDANCE", "- The engine",
                                                     "- LIKE FOR LIKE", "- MEMORY")) + "\n")


def _evidence_prompt(p, return_signal_13f=False, insider_material=False):
    """Persona instructions generated from the evidence permissions: when 13F positions are not a validated return
    signal (and insider trades are immaterial), the ownership key point asks for FACTS only and never for
    'accumulation', 'flows' or a directional read."""
    if return_signal_13f:
        return p
    facts = ("<ownership and insider FACTS only — reported positions and trades, stated as numbers; never a direction, "
             "accumulation/distribution, floor or support; real catalysts vs hype>")
    rep = (('"<13F/insider accumulation; real catalysts>"', f'"{facts}"'),
           ('"<13F/insider positioning; real catalyst vs hype>"', f'"{facts}"'),
           ('"<what 13F/insider positioning and news actually show>"', f'"{facts}"'),
           ("Smart money & news", "Ownership facts & news"),
           ("name the tracked investors whose\n  reported positions rose (positions, never intent), cite real catalysts",
            "state reported positions as facts (never a direction or a return signal), cite real catalysts"),
           ("what positions and trades do the\n  filings show (they record holdings, never intent or motive)",
            "state what the filings record as facts (never a direction, intent or return signal)"),
           ("changes in reported institutional positions, ", ""))
    for a, b in rep:
        p = p.replace(a, b)
    return p


def _guidance_prompt(p, available):
    """Make every guidance instruction conditional on verified company guidance existing in the state."""
    if available:
        rep = (("guidance credibility", "the company's own guidance (cite it only as its [[GUIDANCE_n]] tokens)"),
               ("guidance cut,", "a guidance cut shown by a [[GUIDANCE_n]] token,"))
    else:
        rep = (("guidance credibility", "demand visibility (company guidance is unavailable in canonical data — do not mention guidance)"),
               ("guidance cut, ", ""))
    for a, b in rep:
        p = p.replace(a, b)
    return p


_REBUTTAL_RULES = """
REBUTTAL RULES: "reasoning" = your thesis AFTER hearing the other side, in 1-2 sentences — what
you still believe and what, if anything, you concede. Each key_point: "topic" = the opposing
claim in at most 8 words; "point" = why it fails or survives, settled by one number, and what
that does to your return estimate. No restating your opening case.
"""
_BEAR_SYSTEM_PROMPT += _STYLE_RULES + _ARGUMENT_RULES
_BULL_SYSTEM_PROMPT += _STYLE_RULES + _ARGUMENT_RULES
_NEUTRAL_SYSTEM_PROMPT = _NEUTRAL_SYSTEM_PROMPT + _STYLE_RULES + _ARGUMENT_RULES
_BEAR_REBUTTAL_TEMPLATE = _BEAR_REBUTTAL_TEMPLATE + _REBUTTAL_RULES
_BULL_REBUTTAL_TEMPLATE = _BULL_REBUTTAL_TEMPLATE + _REBUTTAL_RULES

try:
    from predict_weekly import (
        check_earnings_proximity,
        _raw_gpt_judge,
        _raw_kimi_judge,
        _raw_claude_judge,
        _parse_judge_response_weekly,
    )
except ImportError:
    def check_earnings_proximity(data):
        return None, "unknown", ""

    def _raw_gpt_judge(*_a, **_k):
        return None, 0, 0

    def _raw_kimi_judge(*_a, **_k):
        return None, 0, 0

    def _raw_claude_judge(*_a, **_k):
        return None, 0, 0

    def _parse_judge_response_weekly(*_a, **_k):
        return None







# ─── Data Collection (17 endpoints — adds 2Y Treasury) ──────────────────────


def fetch_all_data_yearly(ticker, years=10, no_cache=False):
    """Fetch all data for yearly prediction. 10yr price history + 2Y Treasury."""
    data = {}

    # ── Concurrent cache pre-warm ──────────────────────────────────────────────
    # The 20+ Alpha Vantage endpoints below are independent; fetched one-by-one they
    # serialize network latency (~30s cold). Warm them concurrently first (bounded
    # pool + the locked throttle keeps us under AV's 5/s burst & 75/min caps); the
    # sequential blocks then read straight from the just-written cache. Skipped when
    # no_cache — that path bypasses the cache, so a prewarm would double every call.
    # Specs MUST mirror the (params, cache_key, cache_hours) used by the blocks below.
    if not no_cache:
        _prewarm = [
            ({"function": "OVERVIEW", "symbol": ticker}, f"{ticker}_overview", 168),
            ({"function": "INCOME_STATEMENT", "symbol": ticker}, f"{ticker}_income", 168),
            ({"function": "BALANCE_SHEET", "symbol": ticker}, f"{ticker}_balance", 168),
            ({"function": "CASH_FLOW", "symbol": ticker}, f"{ticker}_cashflow", 168),
            ({"function": "EARNINGS", "symbol": ticker}, f"{ticker}_earnings", 168),
            ({"function": "EARNINGS_ESTIMATES", "symbol": ticker}, f"{ticker}_earnings_estimates", 24),
            ({"function": "GLOBAL_QUOTE", "symbol": ticker}, f"{ticker}_gq", 1),
            ({"function": "INSIDER_TRANSACTIONS", "symbol": ticker}, f"{ticker}_insider", 168),
            ({"function": "REALTIME_OPTIONS", "symbol": ticker}, f"{ticker}_options", 4),
            ({"function": "NEWS_SENTIMENT", "tickers": ticker, "limit": "50", "sort": "LATEST"}, f"{ticker}_sentiment", 2),
            ({"function": "ANALYTICS_FIXED_WINDOW", "SYMBOLS": ticker, "RANGE": "1year",
              "INTERVAL": "DAILY", "OHLCV": "close",
              "CALCULATIONS": "MEAN,STDDEV,CUMULATIVE_RETURN"}, f"{ticker}_analytics_1y", 4),
            ({"function": "TREASURY_YIELD", "interval": "daily", "maturity": "10year"}, "treasury_10y", 12),
            ({"function": "FEDERAL_FUNDS_RATE", "interval": "daily"}, "fed_funds", 12),
            ({"function": "CPI", "interval": "monthly"}, "cpi", 24),
            ({"function": "UNEMPLOYMENT"}, "unemployment", 24),
            ({"function": "TREASURY_YIELD", "interval": "daily", "maturity": "2year"}, "treasury_2y", 12),
            ({"function": "REAL_GDP", "interval": "quarterly"}, "real_gdp", 168),
            ({"function": "NONFARM_PAYROLL"}, "nonfarm_payroll", 48),
            ({"function": "WTI", "interval": "monthly"}, "wti_monthly", 168),
            ({"function": "NATURAL_GAS", "interval": "monthly"}, "natural_gas_monthly", 168),
            ({"function": "COPPER", "interval": "monthly"}, "copper_monthly", 168),
        ]
        from concurrent.futures import ThreadPoolExecutor as _TPE

        def _warm(spec):
            try:
                api_call(spec[0], spec[1], spec[2], no_cache=False)
            except Exception:
                pass

        print(f"    {DIM}Pre-warming {len(_prewarm)} data endpoints (concurrent)…{RESET}", flush=True)
        with _TPE(max_workers=6) as _ex:
            list(_ex.map(_warm, _prewarm))

    # 1. Daily price history (10+ years)
    print(f"    [1/17] Daily price history (10yr)...", end=" ", flush=True)
    cache_file = CACHE_DIR / f"{ticker}_daily.csv"
    cache_meta = CACHE_DIR / f"{ticker}_daily.meta.json"
    if cache_file.exists() and cache_meta.exists():
        try:
            meta = json.loads(cache_meta.read_text())
            if (datetime.now() - datetime.fromisoformat(meta["cached_at"])).total_seconds() < 14400:
                data["daily"] = pd.read_csv(cache_file, index_col=0, parse_dates=True)
                print(f"{GREEN}cached ({len(data['daily'])} days){RESET}")
            else:
                raise Exception("stale")
        except Exception:
            pass

    if "daily" not in data:
        raw = api_call(
            {
                "function": "TIME_SERIES_DAILY_ADJUSTED",
                "symbol": ticker,
                "outputsize": "full",
                "entitlement": "delayed",
            },
            no_cache=no_cache,
        )
        ts = raw.get("Time Series (Daily)", {})
        if ts:
            recs = [{"date": pd.Timestamp(d), "open": float(v["1. open"]), "high": float(v["2. high"]),
                     "low": float(v["3. low"]), "close": float(v["4. close"]),
                     "adj_close": float(v["5. adjusted close"]), "volume": float(v["6. volume"])}
                    for d, v in ts.items()]
            data["daily"] = pd.DataFrame(recs).set_index("date").sort_index()
            data["daily"].to_csv(cache_file)
            cache_meta.write_text(json.dumps({"ticker": ticker, "cached_at": datetime.now().isoformat(),
                                              "rows": len(data["daily"])}))
            print(f"{GREEN}{len(data['daily'])} days{RESET}")
        else:
            print(f"{RED}FAILED{RESET}")
            sys.exit(1)

    cutoff = datetime.now() - timedelta(days=years * 365 + 200)
    data["daily"] = data["daily"][data["daily"].index >= cutoff]
    print(f"    {DIM}Using {len(data['daily'])} days ({years}yr window){RESET}")

    # 2. SPY (market context)
    print(f"    [2/17] SPY market context...", end=" ", flush=True)
    spy_cache = CACHE_DIR / "SPY_daily.csv"
    spy_meta = CACHE_DIR / "SPY_daily.meta.json"
    if spy_cache.exists() and spy_meta.exists():
        try:
            meta = json.loads(spy_meta.read_text())
            if (datetime.now() - datetime.fromisoformat(meta["cached_at"])).total_seconds() < 14400:
                data["spy"] = pd.read_csv(spy_cache, index_col=0, parse_dates=True)
                print(f"{GREEN}cached{RESET}")
            else:
                raise Exception()
        except Exception:
            pass
    if "spy" not in data:
        raw = api_call(
            {
                "function": "TIME_SERIES_DAILY_ADJUSTED",
                "symbol": "SPY",
                "outputsize": "full",
                "entitlement": "delayed",
            },
            no_cache=no_cache,
        )
        ts = raw.get("Time Series (Daily)", {})
        if ts:
            recs = [{"date": pd.Timestamp(d), "adj_close": float(v["5. adjusted close"]),
                     "volume": float(v["6. volume"])} for d, v in ts.items()]
            data["spy"] = pd.DataFrame(recs).set_index("date").sort_index()
            data["spy"].to_csv(spy_cache)
            spy_meta.write_text(json.dumps({"ticker": "SPY", "cached_at": datetime.now().isoformat(),
                                            "rows": len(data["spy"])}))
            print(f"{GREEN}{len(data['spy'])} days{RESET}")
        else:
            print(f"{DIM}unavailable{RESET}")
    if "spy" in data:
        data["spy"] = data["spy"][data["spy"].index >= cutoff]

    # 3. Global Quote
    print(f"    [3/17] Global quote...", end=" ", flush=True)
    gq = api_call({"function": "GLOBAL_QUOTE", "symbol": ticker}, f"{ticker}_gq", 1, no_cache=no_cache)
    gq_data = gq.get("Global Quote", {})
    if gq_data:
        data["quote"] = {k.split(". ")[1] if ". " in k else k: v for k, v in gq_data.items()}
        print(f"{GREEN}${float(data['quote'].get('price', 0)):,.2f}{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 4. Company Overview
    print(f"    [4/17] Company overview...", end=" ", flush=True)
    ov = api_call({"function": "OVERVIEW", "symbol": ticker}, f"{ticker}_overview", 168, no_cache=no_cache)
    if "Symbol" in ov:
        data["overview"] = ov
        pe = ov.get("PERatio", "N/A")
        mc = safe_float(ov.get("MarketCapitalization", 0)) / 1e9
        print(f"{GREEN}P/E:{pe} MktCap:${mc:.1f}B{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 5. Income Statement
    print(f"    [5/17] Income statement...", end=" ", flush=True)
    inc = api_call({"function": "INCOME_STATEMENT", "symbol": ticker}, f"{ticker}_income", 168, no_cache=no_cache)
    if "quarterlyReports" in inc:
        data["income"] = inc
        # Foreign ADRs (Kaspi/KSPI: statements in KZT, price in USD) poison every
        # per-share valuation by the FX rate (~500x). Detect the reporting currency and
        # fetch the rate so valuations can be normalized to the price currency.
        try:
            _rc = (((inc or {}).get("quarterlyReports") or [{}])[0] or {}).get("reportedCurrency")
            if _rc and str(_rc).upper() != "USD":
                _rcU = str(_rc).upper()
                fxr = api_call({"function": "CURRENCY_EXCHANGE_RATE",
                                "from_currency": _rcU, "to_currency": "USD"},
                               f"fx_{_rcU}", 24, no_cache=no_cache)
                _rate = safe_float(((fxr or {}).get("Realtime Currency Exchange Rate") or {})
                                   .get("5. Exchange Rate"))
                if _rate and 0 < _rate < 1000:
                    data["fx_usd"] = float(_rate)
                    data["reported_currency"] = _rcU
                    print(f"    [fx] statements reported in {_rcU}: 1 {_rcU} = {_rate:.6f} USD — valuations normalized")
            if "fx_usd" not in data:
                # reportedCurrency is often NULL for foreign ADRs (KSPI) while the statements
                # are still local-currency. Infer the conversion from two views of the SAME
                # quantity: overview EPS (USD, consistent with PERatio x price) vs trailing
                # EPS from the income statements (local). The ratio doubles as the factor
                # (it also absorbs ADR share-ratio quirks). Engage only on a >=5x mismatch.
                _ovv = data.get("overview") or {}
                _eps_usd = safe_float(_ovv.get("EPS"))
                _sh = safe_float(_ovv.get("SharesOutstanding"))
                _iqs = ((inc or {}).get("quarterlyReports") or [])[:4]
                _ni_ttm = sum((safe_float(q.get("netIncome")) or 0.0) for q in _iqs)
                if _eps_usd and _sh and _sh > 0 and _ni_ttm and len(_iqs) == 4:
                    _eps_stmt = _ni_ttm / _sh
                    if _eps_stmt > 0 and _eps_usd > 0:
                        _ratio = _eps_usd / _eps_stmt
                        if _ratio <= 0.2 or _ratio >= 5.0:
                            data["fx_usd"] = float(_ratio)
                            data["reported_currency"] = "LOCAL (implied)"
                            print(f"    [fx] statement scale differs from USD EPS by {1/_ratio:,.0f}x — "
                                  f"implied conversion {_ratio:.6f} applied to valuations")
        except Exception:
            pass
        rev = float(inc["quarterlyReports"][0].get("totalRevenue", 0)) / 1e9
        print(f"{GREEN}Q rev: ${rev:.1f}B ({len(inc['quarterlyReports'])} quarters){RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 6. Balance Sheet
    print(f"    [6/17] Balance sheet...", end=" ", flush=True)
    bs = api_call({"function": "BALANCE_SHEET", "symbol": ticker}, f"{ticker}_balance", 168, no_cache=no_cache)
    if "quarterlyReports" in bs:
        data["balance"] = bs
        ta = (safe_float(bs["quarterlyReports"][0].get("totalAssets")) or 0) / 1e9
        print(f"{GREEN}assets: ${ta:.1f}B{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 7. Cash Flow
    print(f"    [7/17] Cash flow...", end=" ", flush=True)
    cf = api_call({"function": "CASH_FLOW", "symbol": ticker}, f"{ticker}_cashflow", 168, no_cache=no_cache)
    if "quarterlyReports" in cf:
        data["cashflow"] = cf
        ocf = (safe_float(cf["quarterlyReports"][0].get("operatingCashflow")) or 0) / 1e9
        print(f"{GREEN}OCF: ${ocf:.1f}B{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Convert local-currency statements to USD once at the source (a constant factor,
    # so every ratio and growth rate is unchanged) — this way ALL downstream consumers
    # (historical multiples, fixed DCF, 12M earnings model, story context) see numbers
    # in the price currency. The EARNINGS endpoint is already USD — leave it alone.
    _fxc = float(data.get("fx_usd") or 1.0)
    if _fxc != 1.0:
        def _scale_reports(obj):
            for _sec in ("annualReports", "quarterlyReports"):
                for _rep in (obj or {}).get(_sec, []) or []:
                    for _k, _v in list(_rep.items()):
                        if "shares" in _k.lower():
                            continue                      # share counts stay counts
                        if _k in ("fiscalDateEnding", "reportedCurrency"):
                            continue
                        _f = safe_float(_v)
                        if _f is not None and _f != 0:
                            _rep[_k] = _f * _fxc
        for _tbl in ("income", "balance", "cashflow"):
            if data.get(_tbl):
                _scale_reports(data[_tbl])
        _ovv2 = data.get("overview") or {}
        for _k in ("RevenueTTM", "GrossProfitTTM", "EBITDA", "RevenuePerShareTTM"):
            _f = safe_float(_ovv2.get(_k))
            if _f:
                _ovv2[_k] = _f * _fxc
        print(f"    [fx] statements + overview totals converted to USD at {_fxc:.6f}")

    # 8. Earnings (including forward analyst estimates)
    print(f"    [8/17] Earnings...", end=" ", flush=True)
    earn = api_call({"function": "EARNINGS", "symbol": ticker}, f"{ticker}_earnings", 168, no_cache=no_cache)
    if "quarterlyEarnings" in earn:
        data["earnings"] = earn
        # AV is inconsistent per ticker: this endpoint is USD for some ADRs (TM) but
        # local-currency for others (KSPI). Scale only when THIS payload's TTM EPS
        # disagrees with the USD overview EPS by >=5x.
        _fxe = float(data.get("fx_usd") or 1.0)
        if _fxe != 1.0:
            _eov = safe_float((data.get("overview") or {}).get("EPS"))
            _ttm_rep = sum((safe_float(q.get("reportedEPS")) or 0.0)
                           for q in (earn.get("quarterlyEarnings") or [])[:4])
            if _eov and _eov > 0 and _ttm_rep and (_ttm_rep / _eov >= 5.0 or _ttm_rep / _eov <= 0.2):
                for _sec in ("annualEarnings", "quarterlyEarnings"):
                    for _r in earn.get(_sec) or []:
                        for _k in ("reportedEPS", "estimatedEPS"):
                            _f = safe_float(_r.get(_k))
                            if _f is not None and _f != 0:
                                _r[_k] = _f * _fxe
                print(f"    [fx] EARNINGS payload was statement-currency — EPS history converted to USD")
        qe = earn["quarterlyEarnings"][:4]
        beats = sum(1 for q in qe if safe_float(q.get("surprisePercentage", 0)) > 0)

        forward_quarterly_estimates = []
        for q in earn.get("quarterlyEarnings", []):
            est = safe_float(q.get("estimatedEPS"))
            reported = safe_float(q.get("reportedEPS"))
            if est is not None and est != 0 and (reported is None or reported == 0):
                forward_quarterly_estimates.append({
                    "fiscalDateEnding": q.get("fiscalDateEnding"),
                    "estimatedEPS": est,
                })

        forward_quarterly_estimates = sorted(
            forward_quarterly_estimates,
            key=lambda x: x["fiscalDateEnding"] or "",
        )[:4]

        analyst_fy_eps = None
        if len(forward_quarterly_estimates) >= 2:
            analyst_fy_eps = sum(f["estimatedEPS"] for f in forward_quarterly_estimates)
            if len(forward_quarterly_estimates) < 4:
                analyst_fy_eps = analyst_fy_eps * (4 / len(forward_quarterly_estimates))

        if analyst_fy_eps is None:
            ann = earn.get("annualEarnings", [])
            for a in ann[:3]:
                est = safe_float(a.get("estimatedEPS"))
                rep = safe_float(a.get("reportedEPS"))
                if est is not None and est != 0 and (rep is None or rep == 0):
                    analyst_fy_eps = est
                    break

        data["analyst_consensus_eps"] = analyst_fy_eps
        data["forward_quarterly_estimates"] = forward_quarterly_estimates

        ac_str = f" | consensus FY EPS: ${analyst_fy_eps:.2f}" if analyst_fy_eps else ""
        print(
            f"{GREEN}beats: {beats}/4 ({len(earn['quarterlyEarnings'])} quarters){RESET}"
            f"{DIM}{ac_str}{RESET}"
        )
    else:
        print(f"{DIM}unavailable{RESET}")
        data["analyst_consensus_eps"] = None
        data["forward_quarterly_estimates"] = []

    # 8a. Dedicated analyst consensus (annual EPS + revenue + analyst count)
    print(f"    [+] Earnings estimates (consensus)...", end=" ", flush=True)
    ee_raw = api_call(
        {"function": "EARNINGS_ESTIMATES", "symbol": ticker},
        f"{ticker}_earnings_estimates", 24, no_cache=no_cache,
    )
    _ee_all = ee_raw.get("estimates", []) if isinstance(ee_raw, dict) else []
    _fxee = float(data.get("fx_usd") or 1.0)
    if _fxee != 1.0 and _ee_all:
        _eov2 = safe_float((data.get("overview") or {}).get("EPS"))
        _probe = next((safe_float(e.get("eps_estimate_average")) for e in _ee_all
                       if safe_float(e.get("eps_estimate_average"))), None)
        if _eov2 and _eov2 > 0 and _probe and (_probe / _eov2 >= 5.0 or _probe / _eov2 <= 0.2):
            for _e in _ee_all:
                for _k, _v in list(_e.items()):
                    if _k.startswith(("eps_estimate", "revenue_estimate")):
                        _f = safe_float(_v)
                        if _f is not None and _f != 0:
                            _e[_k] = _f * _fxee
            print(f"[fx→USD] ", end="")
    _today_str = datetime.now().strftime("%Y-%m-%d")
    # Keep only annual-horizon entries with future dates, sorted ascending by date
    _ee_fy = sorted(
        [e for e in _ee_all
         if e.get("horizon") == "fiscal year" and e.get("date", "") >= _today_str],
        key=lambda e: e.get("date", ""),
    )
    _ee_curr = _ee_fy[0] if len(_ee_fy) > 0 else {}
    _ee_next = _ee_fy[1] if len(_ee_fy) > 1 else {}
    _ee_curr_eps = safe_float(_ee_curr.get("eps_estimate_average"), None)

    # 90-day EPS revision drift via timestamped history cache
    _ee_hist_file = CACHE_DIR / f"{ticker}_ee_history.json"
    _ee_hist: list = []
    if _ee_hist_file.exists():
        try:
            _ee_hist = json.loads(_ee_hist_file.read_text())
        except Exception:
            _ee_hist = []
    if _ee_curr_eps is not None:
        _ee_hist.append({"date": datetime.now().isoformat(), "eps": _ee_curr_eps})
        _ee_hist = _ee_hist[-200:]  # keep ~6 months of daily fetches
        try:
            _ee_hist_file.write_text(json.dumps(_ee_hist))
        except Exception:
            pass
    _eps_90d_ago = None
    _now_dt = datetime.now()
    for _entry in reversed(_ee_hist[:-1]):  # scan backwards, skip current entry
        try:
            _entry_dt = datetime.fromisoformat(_entry["date"])
            _days_ago = (_now_dt - _entry_dt).days
            if _days_ago >= 85:  # closest entry at least 85 days ago
                _eps_90d_ago = _entry.get("eps")
                break
        except Exception:
            continue

    data["earnings_estimates_consensus"] = {
        "current_year_eps":      _ee_curr_eps,
        "next_year_eps":         safe_float(_ee_next.get("eps_estimate_average"), None),
        "current_year_eps_low":  safe_float(_ee_curr.get("eps_estimate_low"), None),
        "current_year_eps_high": safe_float(_ee_curr.get("eps_estimate_high"), None),
        "analyst_count":         safe_float(_ee_curr.get("eps_estimate_analyst_count"), None),
        "current_year_revenue":  safe_float(_ee_curr.get("revenue_estimate_average"), None),
        "next_year_revenue":     safe_float(_ee_next.get("revenue_estimate_average"), None),
        "current_year_date":     _ee_curr.get("date"),
        "next_year_date":        _ee_next.get("date"),
        # Revision momentum — how estimates have moved recently
        "eps_7d_ago":            safe_float(_ee_curr.get("eps_estimate_average_7_days_ago"), None),
        "eps_30d_ago":           safe_float(_ee_curr.get("eps_estimate_average_30_days_ago"), None),
        "eps_90d_ago":           _eps_90d_ago,
        "revisions_up_7d":       safe_float(_ee_curr.get("eps_estimate_revision_up_trailing_7_days"), None),
        "revisions_dn_7d":       safe_float(_ee_curr.get("eps_estimate_revision_down_trailing_7_days"), None),
        "revisions_up_30d":      safe_float(_ee_curr.get("eps_estimate_revision_up_trailing_30_days"), None),
        "revisions_dn_30d":      safe_float(_ee_curr.get("eps_estimate_revision_down_trailing_30_days"), None),
    }
    _ee_cy   = data["earnings_estimates_consensus"]["current_year_eps"]
    _ee_n    = int(data["earnings_estimates_consensus"]["analyst_count"] or 0)
    _ee_up7  = data["earnings_estimates_consensus"]["revisions_up_7d"]
    _ee_dn7  = data["earnings_estimates_consensus"]["revisions_dn_7d"]
    if _ee_cy is not None and abs(_ee_cy) > 0.01:
        _rev_str = ""
        if _ee_up7 is not None or _ee_dn7 is not None:
            _up = int(_ee_up7 or 0)
            _dn = int(_ee_dn7 or 0)
            _rev_str = f" ↑{_up}/↓{_dn} 7d"
        print(f"{GREEN}FY EPS ${_ee_cy:.2f} (N={_ee_n} analysts{_rev_str}){RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 9. Insider Transactions
    print(f"    [9/17] Insider transactions...", end=" ", flush=True)
    ins = api_call({"function": "INSIDER_TRANSACTIONS", "symbol": ticker}, f"{ticker}_insider", 168, no_cache=no_cache)
    if "data" in ins and ins["data"]:
        data["insider"] = ins["data"]
        buys = sum(1 for t in ins["data"][:50] if t.get("acquisition_or_disposal") == "A")
        sells = sum(1 for t in ins["data"][:50] if t.get("acquisition_or_disposal") == "D")
        print(f"{GREEN}buys:{buys} sells:{sells}{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 10. Options
    print(f"    [10/17] Options chain...", end=" ", flush=True)
    opts = api_call({"function": "REALTIME_OPTIONS", "symbol": ticker}, f"{ticker}_options", 4, no_cache=no_cache)
    if "data" in opts and opts["data"]:
        data["options"] = opts["data"]
        calls = sum(1 for o in opts["data"] if o.get("type", "").lower() == "call")
        puts = sum(1 for o in opts["data"] if o.get("type", "").lower() == "put")
        print(f"{GREEN}calls:{calls} puts:{puts}{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 11. News Sentiment
    print(f"    [11/17] News sentiment...", end=" ", flush=True)
    ns = api_call({"function": "NEWS_SENTIMENT", "tickers": ticker, "limit": "50", "sort": "LATEST"},
                  f"{ticker}_sentiment", 2, no_cache=no_cache)
    if "feed" in ns:
        data["news"] = ns["feed"]
        scores = []
        news_items = []
        for a in ns["feed"][:50]:
            time_pub = a.get("time_published", "")
            for ts_item in a.get("ticker_sentiment", []):
                if ts_item["ticker"].upper() == ticker.upper():
                    try:
                        sent_score = float(ts_item["ticker_sentiment_score"])
                        relevance = float(ts_item.get("relevance_score", 0.5))
                        scores.append(sent_score)
                        news_items.append({
                            "score": sent_score,
                            "relevance": relevance,
                            "time_published": time_pub,
                        })
                    except (ValueError, TypeError):
                        pass
        data["news_scores"] = scores
        data["news_items"] = news_items
        avg = np.mean(scores) if scores else 0
        c = GREEN if avg > 0.05 else RED if avg < -0.05 else YELLOW
        print(f"{c}{avg:+.4f} ({len(scores)} articles){RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 12. Analytics
    print(f"    [12/17] Analytics...", end=" ", flush=True)
    an = api_call({"function": "ANALYTICS_FIXED_WINDOW", "SYMBOLS": ticker, "RANGE": "1year",
                   "INTERVAL": "DAILY", "OHLCV": "close", "CALCULATIONS": "MEAN,STDDEV,CUMULATIVE_RETURN"},
                  f"{ticker}_analytics_1y", 4, no_cache=no_cache)
    if "payload" in an:
        data["analytics"] = an
        print(f"{GREEN}OK{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 13. Treasury 10Y (full history for ML features)
    print(f"    [13/17] Treasury 10Y...", end=" ", flush=True)
    t10 = api_call({"function": "TREASURY_YIELD", "interval": "daily", "maturity": "10year"},
                   "treasury_10y", 12, no_cache=no_cache)
    if "data" in t10 and t10["data"]:
        data["treasury_10y"] = t10["data"][:60]
        data["treasury_10y_full"] = t10["data"]  # full history for ML
        print(f"{GREEN}{t10['data'][0]['value']}%{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 14. Fed Funds Rate
    print(f"    [14/17] Fed funds rate...", end=" ", flush=True)
    ffr = api_call({"function": "FEDERAL_FUNDS_RATE", "interval": "daily"}, "fed_funds", 12, no_cache=no_cache)
    if "data" in ffr and ffr["data"]:
        data["fed_funds"] = ffr["data"][:60]
        print(f"{GREEN}{ffr['data'][0]['value']}%{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 15. CPI
    print(f"    [15/17] CPI...", end=" ", flush=True)
    cpi = api_call({"function": "CPI", "interval": "monthly"}, "cpi", 24, no_cache=no_cache)
    if "data" in cpi and cpi["data"]:
        data["cpi"] = cpi["data"][:24]
        data["cpi_full"] = cpi["data"]  # full history for ML
        print(f"{GREEN}{cpi['data'][0]['value']}{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 16. Unemployment
    print(f"    [16/17] Unemployment...", end=" ", flush=True)
    unemp = api_call({"function": "UNEMPLOYMENT"}, "unemployment", 24, no_cache=no_cache)
    if "data" in unemp and unemp["data"]:
        data["unemployment"] = unemp["data"][:24]
        data["unemployment_full"] = unemp["data"]  # full history for ML
        print(f"{GREEN}{unemp['data'][0]['value']}%{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # 17. Treasury 2Y (NEW — for yield curve slope)
    print(f"    [17/17] Treasury 2Y (yield curve)...", end=" ", flush=True)
    t2 = api_call({"function": "TREASURY_YIELD", "interval": "daily", "maturity": "2year"},
                  "treasury_2y", 12, no_cache=no_cache)
    if "data" in t2 and t2["data"]:
        data["treasury_2y"] = t2["data"][:60]
        data["treasury_2y_full"] = t2["data"]  # full history for ML
        if data.get("treasury_10y"):
            print(f"{GREEN}{t2['data'][0]['value']}% (slope: {float(t10['data'][0]['value']) - float(t2['data'][0]['value']):+.2f}%){RESET}")
        else:
            print(f"{GREEN}{t2['data'][0]['value']}% (slope: n/a — 10Y unavailable){RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Macro: Real GDP (quarterly, ~30-day publication lag) — cached globally (no ticker prefix)
    print(f"    [+] Real GDP...", end=" ", flush=True)
    gdp_raw = api_call({"function": "REAL_GDP", "interval": "quarterly"}, "real_gdp", 168, no_cache=no_cache)
    data["real_gdp_full"] = gdp_raw.get("data", []) if isinstance(gdp_raw, dict) else []
    if data["real_gdp_full"]:
        _gdp_v = data["real_gdp_full"][0].get("value", "?")
        print(f"{GREEN}{_gdp_v}B{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Macro: Nonfarm Payroll (monthly, ~5-day publication lag) — cached globally
    print(f"    [+] Nonfarm payroll...", end=" ", flush=True)
    nfp_raw = api_call({"function": "NONFARM_PAYROLL"}, "nonfarm_payroll", 48, no_cache=no_cache)
    data["nonfarm_payroll_full"] = nfp_raw.get("data", []) if isinstance(nfp_raw, dict) else []
    if data["nonfarm_payroll_full"]:
        _nfp_v = data["nonfarm_payroll_full"][0].get("value", "?")
        print(f"{GREEN}{_nfp_v}K jobs{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Macro: WTI crude oil (monthly) — cached globally
    print(f"    [+] WTI crude oil...", end=" ", flush=True)
    wti_raw = api_call({"function": "WTI", "interval": "monthly"}, "wti_monthly", 168, no_cache=no_cache)
    data["wti_full"] = wti_raw.get("data", []) if isinstance(wti_raw, dict) else []
    if data["wti_full"]:
        # Use median of 3 most recent valid values to avoid single-month data spikes
        _wti_recent = [safe_float(d["value"]) for d in data["wti_full"][:6]
                       if d.get("value", ".") not in (".", None, "")]
        _wti_recent = [v for v in _wti_recent if v is not None]
        _wti_v = float(np.median(_wti_recent[:3])) if _wti_recent else None
        print(f"{GREEN}${_wti_v:.2f}/bbl{RESET}" if _wti_v else f"{DIM}unavailable{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Macro: Natural gas (monthly) — cached globally; critical for E&P gas producers
    print(f"    [+] Natural gas price...", end=" ", flush=True)
    natgas_raw = api_call({"function": "NATURAL_GAS", "interval": "monthly"}, "natural_gas_monthly", 168, no_cache=no_cache)
    data["natural_gas_full"] = natgas_raw.get("data", []) if isinstance(natgas_raw, dict) else []
    if data["natural_gas_full"]:
        _ng_v = data["natural_gas_full"][0].get("value", "?")
        print(f"{GREEN}${_ng_v}/mmBtu{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Macro: Copper (monthly) — cached globally
    print(f"    [+] Copper price...", end=" ", flush=True)
    copper_raw = api_call({"function": "COPPER", "interval": "monthly"}, "copper_monthly", 168, no_cache=no_cache)
    data["copper_full"] = copper_raw.get("data", []) if isinstance(copper_raw, dict) else []
    if data["copper_full"]:
        _cu_recent = [safe_float(d["value"]) for d in data["copper_full"][:6]
                      if d.get("value", ".") not in (".", None, "")]
        _cu_recent = [v for v in _cu_recent if v is not None]
        _cu_v = float(np.median(_cu_recent[:3])) if _cu_recent else None
        print(f"{GREEN}${_cu_v:,.2f}/mt{RESET}" if _cu_v else f"{DIM}unavailable{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Earnings call transcript (LLM context only) — API requires symbol + quarter (e.g. 2024Q4)
    print(f"    [extra] Earnings transcript...", end=" ", flush=True)
    earnings_data = data.get("earnings", {})
    quarterly = earnings_data.get("quarterlyEarnings", [])
    data["transcript_excerpt"] = None
    if quarterly:
        last_date = quarterly[0].get("fiscalDateEnding", "")
        if last_date:
            try:
                d = pd.Timestamp(last_date)
                quarter_str = f"{d.year}Q{((d.month - 1) // 3) + 1}"
                transcript_raw = api_call(
                    {
                        "function": "EARNINGS_CALL_TRANSCRIPT",
                        "symbol": ticker,
                        "quarter": quarter_str,
                    },
                    cache_key=f"{ticker}_transcript_{quarter_str}",
                    cache_hours=168,
                    no_cache=no_cache,
                )
                transcript_list = transcript_raw.get("transcript", []) if isinstance(transcript_raw, dict) else []
                if transcript_list and isinstance(transcript_list, list):
                    full_text = "\n".join(
                        f"{t.get('speaker', '?')}: {t.get('content', '')}"
                        for t in transcript_list[-20:]
                        if t.get("content")
                    )
                    data["transcript_excerpt"] = full_text[-3000:]
                    print(f"{GREEN}({quarter_str}) {len(full_text)} chars{RESET}")
                else:
                    print(f"{DIM}({quarter_str}) no turns{RESET}")
            except Exception as e:
                print(f"{DIM}unavailable ({e}){RESET}")
        else:
            print(f"{DIM}no fiscal date{RESET}")
    else:
        print(f"{DIM}no earnings series{RESET}")

    # Institutional 13F-style holdings: not available via Alpha Vantage public API.
    # _institutional_net_buying ML features are set to neutral in build_ml_features_yearly.
    data["institutional"] = []

    # Shares outstanding (buyback rate)
    print(f"    [extra] Shares outstanding...", end=" ", flush=True)
    sh_raw = api_call(
        {"function": "SHARES_OUTSTANDING", "symbol": ticker},
        f"{ticker}_shares_out",
        24,
        no_cache=no_cache,
    )
    data["shares_outstanding"] = sh_raw.get("data", [])
    print(f"{GREEN}{len(data['shares_outstanding'])} rows{RESET}" if data["shares_outstanding"] else f"{DIM}unavailable{RESET}")

    # Dividend history (growth rate + streak)
    print(f"    [+] Dividend history...", end=" ", flush=True)
    div_raw = api_call(
        {"function": "DIVIDENDS", "symbol": ticker},
        f"{ticker}_dividends", 168, no_cache=no_cache,
    )
    data["dividends"] = div_raw.get("data", []) if isinstance(div_raw, dict) else []
    if data["dividends"]:
        print(f"{GREEN}{len(data['dividends'])} payments{RESET}")
    else:
        print(f"{DIM}unavailable (non-dividend payer){RESET}")

    # Historical per-ticker put/call ratio (options sentiment)
    print(f"    [+] Historical put/call ratio...", end=" ", flush=True)
    pcr_raw = api_call(
        {"function": "HISTORICAL_PUT_CALL_RATIO", "symbol": ticker},
        f"{ticker}_put_call_hist", 24, no_cache=no_cache,
    )
    # API returns put_call_ratio_full_chain (full-chain current snapshot) + by_expiration list
    _pcr_full = safe_float(pcr_raw.get("put_call_ratio_full_chain")) if isinstance(pcr_raw, dict) else None
    _pcr_by_exp = pcr_raw.get("put_call_ratio_by_expiration", []) if isinstance(pcr_raw, dict) else []
    data["put_call_ratio_current"] = _pcr_full
    data["put_call_ratio_hist"] = _pcr_by_exp  # expiration-bucketed snapshot, not time series
    if _pcr_full is not None:
        print(f"{GREEN}P/C (full chain): {_pcr_full:.2f}{RESET}")
    else:
        print(f"{DIM}unavailable{RESET}")

    # Historical options (IV)
    print(f"    [extra] Historical options...", end=" ", flush=True)
    ho_raw = api_call(
        {"function": "HISTORICAL_OPTIONS", "symbol": ticker},
        f"{ticker}_hist_options",
        8,
        no_cache=no_cache,
    )
    data["historical_options"] = ho_raw.get("data", [])
    print(f"{GREEN}{len(data['historical_options'])} rows{RESET}" if data["historical_options"] else f"{DIM}unavailable{RESET}")

    # Earnings calendar (12 months) — CSV response from API
    print(f"    [extra] Earnings calendar (12m)...", end=" ", flush=True)
    earn_cal_raw = api_call(
        {"function": "EARNINGS_CALENDAR", "symbol": ticker, "horizon": "12month"},
        f"{ticker}_earn_cal_12m",
        24,
        no_cache=no_cache,
        csv_response=True,
    )
    cal_text = earn_cal_raw if isinstance(earn_cal_raw, str) else (
        earn_cal_raw.get("csv", "") if isinstance(earn_cal_raw, dict) else ""
    )
    if cal_text and cal_text.strip() and not cal_text.lstrip().startswith("{"):
        try:
            cal_df = pd.read_csv(io.StringIO(cal_text))
            recs = cal_df.to_dict("records")
            data["earnings_calendar"] = recs
            data["earnings_calendar_12m"] = recs
            n_ev = len(recs)
            data["n_earnings_events_12m"] = n_ev
            if recs:
                try:
                    nxt = recs[0]
                    ed = nxt.get("reportDate") or nxt.get("date")
                    if ed:
                        data["days_to_next_earnings"] = max(
                            0, (pd.Timestamp(ed) - pd.Timestamp.now()).days
                        )
                except Exception:
                    pass
            print(f"{GREEN}{n_ev} events{RESET}")
        except Exception:
            data["earnings_calendar"] = []
            data["earnings_calendar_12m"] = []
            data["n_earnings_events_12m"] = 0
            print(f"{DIM}parse failed{RESET}")
    else:
        data["earnings_calendar"] = []
        data["earnings_calendar_12m"] = []
        data["n_earnings_events_12m"] = 0
        print(f"{DIM}unavailable{RESET}")

    # Short interest via yfinance (Alpha Vantage has no short interest endpoint)
    print(f"    [extra] Short interest...", end=" ", flush=True)
    try:
        _si_cf = CACHE_DIR / f"{ticker}_short_interest_yearly.json"
        _si_mf = CACHE_DIR / f"{ticker}_short_interest_yearly.meta.json"
        _si_pct = None
        _si_from_cache = False
        if not no_cache and _si_cf.exists() and _si_mf.exists():
            try:
                _si_age = (datetime.now() - datetime.fromisoformat(
                    json.loads(_si_mf.read_text())["cached_at"]
                )).total_seconds()
                if _si_age < 86400:
                    _si_pct = json.loads(_si_cf.read_text()).get("shortPercentOfFloat")
                    _si_from_cache = True
            except Exception:
                pass
        if not _si_from_cache:
            import yfinance as _yf
            import contextlib, io as _io
            _yf_sink = _io.StringIO()
            with contextlib.redirect_stderr(_yf_sink):
                _yf_info = _yf.Ticker(ticker).info
            _si_pct = _yf_info.get("shortPercentOfFloat")
            _si_cf.write_text(json.dumps({"shortPercentOfFloat": _si_pct}))
            _si_mf.write_text(json.dumps({"cached_at": datetime.now().isoformat()}))
        data["short_interest_pct_float"] = float(_si_pct) if _si_pct is not None else None
        if data["short_interest_pct_float"] is not None:
            print(f"{GREEN}{data['short_interest_pct_float']:.1%} of float{RESET}")
        else:
            print(f"{DIM}none{RESET}")
    except Exception:
        data["short_interest_pct_float"] = None
        print(f"{DIM}unavailable{RESET}")

    # VIX (INDEX_DATA)
    print(f"    [extra] VIX index...", end=" ", flush=True)
    vix_raw = api_call(
        {"function": "INDEX_DATA", "symbol": "VIX", "interval": "daily"},
        "VIX_index_daily_yearly",
        12,
        no_cache=no_cache,
    )
    vix_series = vix_raw.get("data") or []
    if vix_series:
        vrecs = []
        for row in vix_series:
            try:
                d = row.get("date") or row.get("timestamp")
                c = float(row.get("close", row.get("4. close", row.get("value", 0))))
                if d and c > 0:
                    vrecs.append({"date": pd.Timestamp(d), "vix_close": c})
            except (ValueError, TypeError):
                continue
        if vrecs:
            data["vix_df"] = pd.DataFrame(vrecs).set_index("date").sort_index()
            print(f"{GREEN}{len(data['vix_df'])} rows{RESET}")
        else:
            data["vix_df"] = None
            print(f"{DIM}unavailable{RESET}")
    else:
        data["vix_df"] = None
        print(f"{DIM}unavailable{RESET}")

    return data


# ─── Point-in-Time Fundamental Feature Builder ──────────────────────────────


def _map_report_dates(data):
    """Build fiscalDateEnding → reportedDate mapping.
    Uses earnings.reportedDate where available, falls back to fiscalDateEnding + 45 days."""
    mapping = {}
    if "earnings" in data and "quarterlyEarnings" in data["earnings"]:
        for q in data["earnings"]["quarterlyEarnings"]:
            fde = q.get("fiscalDateEnding", "")
            rd = q.get("reportedDate", "")
            if fde and rd:
                mapping[fde] = rd
    return mapping


def _compute_ttm_from_reports(reports, field, end_idx, n=4):
    """Sum trailing n quarters starting from end_idx."""
    total = 0.0
    count = 0
    for i in range(end_idx, min(end_idx + n, len(reports))):
        v = safe_float(reports[i].get(field, 0))
        if v != 0:
            total += v
            count += 1
    return total if count >= max(n - 1, 1) else None


def build_pit_fundamental_features(daily_index, data):
    """Build point-in-time fundamental features aligned to daily price dates.

    For each trading date, uses the most recent quarterly report whose
    reportedDate is BEFORE that date. Prevents look-ahead bias.
    Price-dependent ratios (P/E, P/FCF) use daily closing prices.
    """
    report_date_map = _map_report_dates(data)
    inc_q = data.get("income", {}).get("quarterlyReports", [])
    bs_q = data.get("balance", {}).get("quarterlyReports", [])
    cf_q = data.get("cashflow", {}).get("quarterlyReports", [])
    ov = data.get("overview", {})
    shares = safe_float(ov.get("SharesOutstanding", 0))

    if not inc_q or not bs_q or not cf_q:
        return None

    # Build report timeline: list of (available_date, report_index) sorted by available_date
    report_timeline = []
    for idx, report in enumerate(inc_q):
        fde = report.get("fiscalDateEnding", "")
        if not fde:
            continue
        # When was this report available to investors?
        if fde in report_date_map:
            avail = pd.Timestamp(report_date_map[fde])
        else:
            avail = pd.Timestamp(fde) + pd.Timedelta(days=45)  # assume 45-day filing lag
        report_timeline.append((avail, idx))

    report_timeline.sort(key=lambda x: x[0])

    # For each daily date, find which report was most recently available
    pit_data = []
    daily_close = data.get("daily", pd.DataFrame())
    close_col = "adj_close" if "adj_close" in daily_close.columns else daily_close.columns[0] if len(daily_close.columns) > 0 else None

    for dt in daily_index:
        row = {"date": dt}
        # Find most recent available report
        best_idx = None
        for avail_date, rep_idx in report_timeline:
            if avail_date <= dt:
                best_idx = rep_idx
            else:
                break

        if best_idx is None:
            pit_data.append(row)
            continue

        # Get daily price for price-dependent ratios
        price = None
        if close_col and dt in daily_close.index:
            price = float(daily_close.loc[dt, close_col])

        # TTM computations from best_idx (most recent available quarter)
        ttm_revenue = _compute_ttm_from_reports(inc_q, "totalRevenue", best_idx)
        ttm_net_income = _compute_ttm_from_reports(inc_q, "netIncome", best_idx)
        ttm_ebit = _compute_ttm_from_reports(inc_q, "ebit", best_idx)
        ttm_op_income = _compute_ttm_from_reports(inc_q, "operatingIncome", best_idx)
        ttm_interest = _compute_ttm_from_reports(inc_q, "interestExpense", best_idx)

        ttm_ocf = _compute_ttm_from_reports(cf_q, "operatingCashflow", best_idx) if best_idx < len(cf_q) else None
        ttm_capex = _compute_ttm_from_reports(cf_q, "capitalExpenditures", best_idx) if best_idx < len(cf_q) else None
        ttm_fcf = (ttm_ocf - abs(ttm_capex)) if ttm_ocf is not None and ttm_capex is not None else None

        # Balance sheet (snapshot, not TTM)
        bs_idx = min(best_idx, len(bs_q) - 1) if bs_q else None
        equity = safe_float(bs_q[bs_idx].get("totalShareholderEquity", 0)) if bs_idx is not None else None
        total_debt = None
        if bs_idx is not None:
            td = safe_float(bs_q[bs_idx].get("shortLongTermDebtTotal", 0))
            if td == 0:
                td = safe_float(bs_q[bs_idx].get("shortTermDebt", 0)) + safe_float(bs_q[bs_idx].get("longTermDebt", 0))
            total_debt = td if td > 0 else 0

        # ── Price-dependent valuation ratios ──
        if price and price > 0 and shares > 0:
            mkt_cap = price * shares
            if ttm_net_income and ttm_net_income > 0:
                row["pit_pe"] = mkt_cap / ttm_net_income
            if ttm_fcf and ttm_fcf > 0:
                row["pit_p_fcf"] = mkt_cap / ttm_fcf

        # ── Profitability ──
        if ttm_net_income is not None and equity and equity > 0:
            row["pit_roe"] = ttm_net_income / equity
        if ttm_ebit is not None and equity is not None and total_debt is not None:
            invested = equity + total_debt
            if invested > 0:
                row["pit_roic"] = ttm_ebit * 0.79 / invested  # NOPAT = EBIT * (1-tax)
        if ttm_op_income is not None and ttm_revenue and ttm_revenue > 0:
            row["pit_operating_margin"] = ttm_op_income / ttm_revenue
        if ttm_fcf is not None and ttm_revenue and ttm_revenue > 0:
            row["pit_fcf_margin"] = ttm_fcf / ttm_revenue

        # ── Financial health ──
        if equity and equity > 0 and total_debt is not None:
            row["pit_d_e"] = total_debt / equity
        if ttm_ebit and ttm_interest and ttm_interest > 0:
            row["pit_interest_coverage"] = ttm_ebit / ttm_interest
        elif ttm_ebit and ttm_ebit > 0 and (ttm_interest is None or ttm_interest == 0):
            row["pit_interest_coverage"] = 20.0  # cap for debt-free

        # ── Growth (YoY) ──
        prev_idx = best_idx + 4  # same quarter, prior year
        if prev_idx < len(inc_q):
            prev_revenue = _compute_ttm_from_reports(inc_q, "totalRevenue", prev_idx)
            prev_ni = _compute_ttm_from_reports(inc_q, "netIncome", prev_idx)
            if prev_revenue and prev_revenue > 0 and ttm_revenue:
                row["pit_revenue_growth"] = (ttm_revenue - prev_revenue) / abs(prev_revenue)
            if prev_ni and abs(prev_ni) > 0 and ttm_net_income is not None:
                row["pit_earnings_growth"] = (ttm_net_income - prev_ni) / abs(prev_ni)

        if prev_idx < len(cf_q) and ttm_fcf is not None:
            prev_ocf = _compute_ttm_from_reports(cf_q, "operatingCashflow", prev_idx)
            prev_capex = _compute_ttm_from_reports(cf_q, "capitalExpenditures", prev_idx)
            if prev_ocf is not None and prev_capex is not None:
                prev_fcf = prev_ocf - abs(prev_capex)
                if abs(prev_fcf) > 0:
                    row["pit_fcf_growth"] = (ttm_fcf - prev_fcf) / abs(prev_fcf)

        total_assets = (
            safe_float(bs_q[bs_idx].get("totalAssets", 0)) if bs_idx is not None else None
        )
        ttm_gross_profit = _compute_ttm_from_reports(inc_q, "grossProfit", best_idx)

        if equity and equity > 0 and price and price > 0 and shares > 0:
            mkt_cap = price * shares
            row["pit_log_bm"] = np.log(max(equity, 1.0) / max(mkt_cap, 1.0))

        if ttm_net_income is not None and price and price > 0 and shares > 0:
            row["pit_ep"] = ttm_net_income / (price * shares)

        if ttm_revenue and ttm_revenue > 0 and price and price > 0 and shares > 0:
            row["pit_sp"] = ttm_revenue / (price * shares)

        if ttm_gross_profit and total_assets and total_assets > 0:
            row["pit_gross_prof_assets"] = ttm_gross_profit / total_assets

        if ttm_net_income is not None and ttm_ocf is not None and total_assets and total_assets > 0:
            row["pit_accruals"] = (ttm_net_income - ttm_ocf) / total_assets

        if bs_idx is not None and total_assets and total_assets > 0:
            prev_bs_idx = min(bs_idx + 4, len(bs_q) - 1)
            if prev_bs_idx != bs_idx:
                prev_ta = safe_float(bs_q[prev_bs_idx].get("totalAssets", 0))
                if prev_ta and prev_ta > 0:
                    row["pit_asset_growth"] = total_assets / prev_ta - 1

        if total_debt is not None and total_assets and total_assets > 0:
            row["pit_debt_to_assets"] = total_debt / total_assets

        _de = row.get("pit_d_e", 0)
        _ic = row.get("pit_interest_coverage", 20)
        _pm = row.get("pit_operating_margin", 0)
        dr = 0.0
        if _de > 2.0:
            dr += 0.4
        elif _de > 1.0:
            dr += 0.2
        if _ic < 2.0:
            dr += 0.3
        elif _ic < 5.0:
            dr += 0.1
        if _pm is not None and _pm < 0:
            dr += 0.3
        row["pit_distress_prob"] = min(dr, 1.0)

        pit_data.append(row)

    pit_df = pd.DataFrame(pit_data).set_index("date")
    # Forward-fill: fundamentals don't change between reports
    pit_df = pit_df.ffill()
    return pit_df


def build_pit_macro_features(daily_index, data):
    """Build point-in-time macro features aligned to daily dates.
    10Y yield and 2Y yield are daily; CPI and unemployment are monthly with ~15 day pub lag."""

    macro_df = pd.DataFrame(index=daily_index)

    # 10Y Treasury yield (daily)
    t10_full = data.get("treasury_10y_full", [])
    if t10_full:
        t10_series = pd.Series(
            {pd.Timestamp(d["date"]): safe_float(d["value"]) for d in t10_full if d.get("value", ".") != "."},
            dtype=float
        ).sort_index()
        macro_df["macro_10y_yield"] = t10_series.reindex(daily_index, method="ffill")

    # 2Y Treasury yield (daily) → yield curve slope
    t2_full = data.get("treasury_2y_full", [])
    if t2_full:
        t2_series = pd.Series(
            {pd.Timestamp(d["date"]): safe_float(d["value"]) for d in t2_full if d.get("value", ".") != "."},
            dtype=float
        ).sort_index()
        macro_df["macro_2y_yield"] = t2_series.reindex(daily_index, method="ffill")
        if "macro_10y_yield" in macro_df.columns:
            macro_df["macro_yield_slope"] = macro_df["macro_10y_yield"] - macro_df["macro_2y_yield"]

    # CPI (monthly, ~15 day publication lag)
    cpi_full = data.get("cpi_full", [])
    if cpi_full:
        cpi_series = pd.Series(
            {pd.Timestamp(d["date"]) + pd.Timedelta(days=15): safe_float(d["value"])
             for d in cpi_full if d.get("value", ".") != "."},
            dtype=float
        ).sort_index()
        macro_df["macro_cpi"] = cpi_series.reindex(daily_index, method="ffill")
        # CPI YoY change
        macro_df["macro_cpi_yoy"] = macro_df["macro_cpi"].pct_change(252)  # approximate YoY

    # Unemployment (monthly, ~15 day publication lag)
    unemp_full = data.get("unemployment_full", [])
    if unemp_full:
        unemp_series = pd.Series(
            {pd.Timestamp(d["date"]) + pd.Timedelta(days=15): safe_float(d["value"])
             for d in unemp_full if d.get("value", ".") != "."},
            dtype=float
        ).sort_index()
        macro_df["macro_unemployment"] = unemp_series.reindex(daily_index, method="ffill")

    # Real GDP (quarterly, ~30-day publication lag)
    gdp_full = data.get("real_gdp_full", [])
    if gdp_full:
        gdp_series = pd.Series(
            {pd.Timestamp(d["date"]) + pd.Timedelta(days=30): safe_float(d["value"])
             for d in gdp_full if d.get("value", ".") not in (".", None, "")},
            dtype=float,
        ).sort_index()
        macro_df["macro_gdp_level"] = gdp_series.reindex(daily_index, method="ffill")
        # YoY growth: GDP is quarterly, reindexed daily → use 252-day pct_change
        macro_df["macro_gdp_growth_yoy"] = macro_df["macro_gdp_level"].pct_change(252)
        # QoQ growth (~63 trading days per quarter)
        macro_df["macro_gdp_growth_qoq"] = macro_df["macro_gdp_level"].pct_change(63)

    # Nonfarm Payroll (monthly, ~5-day publication lag) — absolute MoM change in thousands
    nfp_full = data.get("nonfarm_payroll_full", [])
    if nfp_full:
        nfp_series = pd.Series(
            {pd.Timestamp(d["date"]) + pd.Timedelta(days=5): safe_float(d["value"])
             for d in nfp_full if d.get("value", ".") not in (".", None, "")},
            dtype=float,
        ).sort_index()
        macro_df["macro_nonfarm_level"] = nfp_series.reindex(daily_index, method="ffill")
        # Month-over-month absolute change (net jobs added, in thousands)
        macro_df["macro_nonfarm_mom"] = macro_df["macro_nonfarm_level"].diff(21)

    # WTI crude oil (monthly) — YoY price change as inflation/cost proxy
    wti_full = data.get("wti_full", [])
    if wti_full:
        wti_series = pd.Series(
            {pd.Timestamp(d["date"]): safe_float(d["value"])
             for d in wti_full if d.get("value", ".") not in (".", None, "")},
            dtype=float,
        ).sort_index()
        # Clip rolling z-score outliers (e.g. single-month data errors)
        if len(wti_series) >= 12:
            _wti_med = wti_series.rolling(12, min_periods=6).median()
            _wti_std = wti_series.rolling(12, min_periods=6).std().fillna(wti_series.std())
            _wti_z = (wti_series - _wti_med) / (_wti_std + 1e-9)
            wti_series = wti_series.where(abs(_wti_z) < 3.0, _wti_med)
        macro_df["macro_wti"] = wti_series.reindex(daily_index, method="ffill")
        macro_df["macro_wti_yoy"] = macro_df["macro_wti"].pct_change(252)

    # Copper (monthly) — YoY price change as global growth barometer
    copper_full = data.get("copper_full", [])
    if copper_full:
        copper_series = pd.Series(
            {pd.Timestamp(d["date"]): safe_float(d["value"])
             for d in copper_full if d.get("value", ".") not in (".", None, "")},
            dtype=float,
        ).sort_index()
        if len(copper_series) >= 12:
            _cu_med = copper_series.rolling(12, min_periods=6).median()
            _cu_std = copper_series.rolling(12, min_periods=6).std().fillna(copper_series.std())
            _cu_z = (copper_series - _cu_med) / (_cu_std + 1e-9)
            copper_series = copper_series.where(abs(_cu_z) < 3.0, _cu_med)
        macro_df["macro_copper"] = copper_series.reindex(daily_index, method="ffill")
        macro_df["macro_copper_yoy"] = macro_df["macro_copper"].pct_change(252)

    # Natural gas (monthly) — Henry Hub spot; primary driver for E&P gas producers
    natgas_full = data.get("natural_gas_full", [])
    if natgas_full:
        natgas_series = pd.Series(
            {pd.Timestamp(d["date"]): safe_float(d["value"])
             for d in natgas_full if d.get("value", ".") not in (".", None, "")},
            dtype=float,
        ).sort_index()
        macro_df["macro_natgas"] = natgas_series.reindex(daily_index, method="ffill")
        macro_df["macro_natgas_yoy"] = macro_df["macro_natgas"].pct_change(252)

    return macro_df


# ─── Yearly Technical Analysis ──────────────────────────────────────────────


def analyze_technicals_yearly(daily, spy_df=None):
    """Technical signals calibrated for 1-year horizon.
    Emphasizes mean-reversion and long-term trend, de-emphasizes short-term momentum."""
    close = daily["adj_close"]

    signals = {}

    # Long-term trend
    sma50 = close.rolling(50).mean().iloc[-1]
    sma200 = close.rolling(200).mean().iloc[-1]
    current = close.iloc[-1]
    signals["trend_sma50"] = 1 if current > sma50 else -1
    signals["trend_sma200"] = 1 if current > sma200 else -1
    signals["golden_cross"] = 1 if sma50 > sma200 else -1

    # SMA200 slope (long-term trend direction)
    sma200_series = close.rolling(200).mean()
    if len(sma200_series.dropna()) > 63:
        sma200_slope = (sma200_series.iloc[-1] / sma200_series.iloc[-63] - 1)
        signals["sma200_slope_3m"] = sma200_slope

    # Momentum (longer periods matter more for yearly)
    mom_3m = current / close.iloc[-63] - 1 if len(close) > 63 else 0
    mom_6m = current / close.iloc[-126] - 1 if len(close) > 126 else 0
    mom_12m = current / close.iloc[-252] - 1 if len(close) > 252 else 0
    signals["momentum_3m"] = mom_3m
    signals["momentum_6m"] = mom_6m
    signals["momentum_12m"] = mom_12m

    # RSI (mean-reversion signal — more powerful at 1yr)
    delta = close.diff()
    gain = delta.where(delta > 0, 0.0).ewm(com=13, min_periods=14).mean()
    loss = -delta.where(delta < 0, 0.0).ewm(com=13, min_periods=14).mean()
    rs = gain / loss.replace(0, np.nan)
    rsi = (100 - 100 / (1 + rs)).iloc[-1]
    signals["rsi_14"] = rsi

    # Bollinger %B
    sma20 = close.rolling(20).mean()
    std20 = close.rolling(20).std()
    bb = (close - (sma20 - 2 * std20)) / (4 * std20)
    signals["bb_pct_b"] = bb.iloc[-1]

    # Volatility
    ret = close.pct_change()
    signals["volatility_20d"] = ret.rolling(20).std().iloc[-1]
    signals["volatility_63d"] = ret.rolling(63).std().iloc[-1]
    signals["volatility_252d"] = ret.rolling(252).std().iloc[-1] if len(ret) > 252 else None

    # 52-week position (mean-reversion: far from high = bullish long-term)
    high_252 = close.rolling(252).max().iloc[-1] if len(close) > 252 else close.max()
    low_252 = close.rolling(252).min().iloc[-1] if len(close) > 252 else close.min()
    signals["dist_52w_high"] = current / high_252 - 1
    signals["dist_52w_low"] = current / low_252 - 1

    # Historical yearly return (base rate)
    if len(close) > 504:
        yearly_rets = close.pct_change(252).dropna()
        avg_yearly = yearly_rets.mean()
        signals["avg_yearly_return"] = avg_yearly
    else:
        avg_yearly = 0.10  # assume market average
        signals["avg_yearly_return"] = avg_yearly

    # ── Score (yearly-calibrated) ──
    score = 0.0
    n = 0

    # Long-term trend (low weight at 1yr — price reverts)
    if signals["trend_sma200"] > 0 and signals.get("sma200_slope_3m", 0) > 0:
        score += 0.1
    elif signals["trend_sma200"] < 0 and signals.get("sma200_slope_3m", 0) < 0:
        score -= 0.1
    n += 1

    # 12-month momentum (academic evidence: works 3-12m, then reverses)
    if mom_12m > 0.3:
        score -= 0.05  # mean-reversion risk after big run
    elif mom_12m > 0.1:
        score += 0.1   # moderate momentum is bullish
    elif mom_12m < -0.3:
        score += 0.15  # deep drawdown = mean-reversion opportunity
    elif mom_12m < -0.1:
        score -= 0.05  # mild downtrend, unclear
    n += 1

    # RSI mean-reversion (stronger signal at yearly horizon)
    if rsi < 30:
        score += 0.2   # oversold = strong mean-reversion buy
    elif rsi < 40:
        score += 0.1
    elif rsi > 70:
        score -= 0.15  # overbought = risk of reversion
    elif rsi > 60:
        score -= 0.05
    n += 1

    # 52-week high distance (bigger drops = better long-term entry)
    dist_high = signals["dist_52w_high"]
    if dist_high < -0.40:
        score += 0.2
    elif dist_high < -0.20:
        score += 0.1
    elif dist_high > -0.05:
        score -= 0.05  # near highs, less upside
    n += 1

    norm_score = score / max(n * 0.1, 1)
    expected_return = np.clip(norm_score * 0.15, -0.25, 0.25)  # wider range for yearly

    return {
        "expected_return": expected_return,
        "confidence": min(n / 4, 0.5),  # low confidence — technicals weak at 1yr
        "signals": signals,
    }


# ─── Yearly Macro Analysis ──────────────────────────────────────────────────


def analyze_macro_yearly(data):
    """Macro analysis calibrated for 1-year horizon. Adds yield curve signal."""
    signals = {}
    score = 0.0
    n = 0

    # Treasury 10Y
    t10 = data.get("treasury_10y", [])
    if t10 and len(t10) >= 2:
        rate = safe_float(t10[0]["value"])
        signals["treasury_10y"] = rate
        if len(t10) >= 20:
            prev = safe_float(t10[19]["value"])
            change = rate - prev
            signals["treasury_10y_change_20d"] = change
            if change < -0.30:
                score += 0.15  # falling rates = bullish
            elif change > 0.30:
                score -= 0.10  # rising rates = headwind

        # Level-based: very high rates = headwind, very low = tailwind
        if rate > 5.0:
            score -= 0.1
        elif rate < 3.0:
            score += 0.1
        n += 1

    # Yield curve slope (10Y - 2Y) — KEY yearly signal
    t2 = data.get("treasury_2y", [])
    if t2 and t10:
        rate_10y = safe_float(t10[0]["value"])
        rate_2y = safe_float(t2[0]["value"])
        slope = rate_10y - rate_2y
        signals["yield_curve_slope"] = slope
        if slope < -0.5:
            score -= 0.20  # deeply inverted = recession risk
            signals["yield_curve_signal"] = "INVERTED (recession risk)"
        elif slope < 0:
            score -= 0.10  # mildly inverted
            signals["yield_curve_signal"] = "INVERTED (mild)"
        elif slope > 1.0:
            score += 0.10  # steep curve = expansionary
            signals["yield_curve_signal"] = "STEEP (expansionary)"
        else:
            signals["yield_curve_signal"] = "NORMAL"
        n += 1

    # Fed Funds Rate
    ffr = data.get("fed_funds", [])
    if ffr:
        rate = safe_float(ffr[0]["value"])
        signals["fed_funds_rate"] = rate
        # Direction matters more than level for yearly
        if len(ffr) >= 60:
            prev = safe_float(ffr[59]["value"])
            if rate < prev - 0.5:
                score += 0.15  # rate cutting cycle = bullish
                signals["fed_direction"] = "CUTTING"
            elif rate > prev + 0.5:
                score -= 0.10  # tightening cycle = headwind
                signals["fed_direction"] = "HIKING"
            else:
                signals["fed_direction"] = "STABLE"
        n += 1

    # CPI
    cpi_data = data.get("cpi", [])
    if cpi_data and len(cpi_data) >= 2:
        current_cpi = safe_float(cpi_data[0]["value"])
        prev_cpi = safe_float(cpi_data[1]["value"])
        signals["cpi"] = current_cpi
        if prev_cpi > 0:
            mom = (current_cpi - prev_cpi) / prev_cpi
            signals["cpi_change_mom"] = mom
        # YoY CPI (12 months ago)
        if len(cpi_data) >= 13:
            cpi_12m = safe_float(cpi_data[12]["value"])
            if cpi_12m > 0:
                cpi_yoy = (current_cpi - cpi_12m) / cpi_12m
                signals["cpi_yoy"] = cpi_yoy
                if cpi_yoy > 0.05:
                    score -= 0.10  # high inflation = headwind
                elif cpi_yoy < 0.02:
                    score += 0.05  # low inflation = tailwind
        n += 1

    # Unemployment
    unemp = data.get("unemployment", [])
    if unemp and len(unemp) >= 2:
        rate = safe_float(unemp[0]["value"])
        signals["unemployment"] = rate
        if len(unemp) >= 12:
            prev_12m = safe_float(unemp[11]["value"])
            change = rate - prev_12m
            signals["unemployment_change_12m"] = change
            if change > 1.0:
                score -= 0.15  # rapidly rising = recession
            elif change < -0.5:
                score += 0.10  # improving labor market
        n += 1

    if n == 0:
        return None

    norm = score / max(n * 0.1, 1)
    expected_return = np.clip(norm * 0.12, -0.15, 0.15)

    return {
        "expected_return": expected_return,
        "confidence": min(n / 5, 1.0),
        "signals": signals,
    }


# ─── ML Feature Builder (Price + Fundamental + Macro) ───────────────────────


def build_ml_features_yearly(daily, spy_df=None, pit_fund=None, pit_macro=None, extra=None):
    """Build features for 252-day return prediction.
    Three feature groups: price-derived + fundamental + macro.

    For companies with limited history (e.g. recent IPOs with 8+ quarters),
    long-window lookback features are capped adaptively so that clean rows
    (rows with both a valid feature AND a valid 252-day forward target) always
    exist.  Rule: only include a feature with N-day lookback when
    N <= max_lookback = len(close) - 252 - 10.
    """
    close = daily["adj_close"]
    feat = pd.DataFrame(index=daily.index)

    # Max lookback that still leaves ≥100 clean rows after the 252-day target consumes
    # the tail.  Formula: n_days - 252 (target) - 100 (min clean buffer).
    # This eliminates a hole at ~520-619 days where a buffer of 10 would include
    # return_252d but leave < 100 clean rows when combined with the 252-day target.
    # E.g. 480-day stock → max_lookback=128; 600-day → 248; 1083-day → 731.
    max_lookback = max(21, len(close) - 252 - 100)

    # ── A. Price-derived features — LONG-HORIZON ONLY ──
    # Pruned 2026: dropped short-horizon features (rsi_14, reversal_1m, return_21d/63d,
    # downside_vol_63d, amihud_63d, turnover_ratio, skew_63d, rolling_beta_63d).
    # At 252-day target, these have near-zero IC and only add fitting degrees of freedom
    # against ~10 independent yearly OOS samples. Long-window analogues (252d) are kept
    # because they capture risk priced in the cross-section (low-vol anomaly, beta, idio vol).
    # 12-1 momentum is kept — Jegadeesh-Titman remains the highest-IC long-horizon price factor.
    # Tier-1 prune (2026): dropped vol_252d (redundant with downside_vol/idio_vol),
    # skew_252d (low IC at 12m), and spy_return_252d (market-wide; already encoded in
    # relative_252d = stock − SPY).
    ret = close.pct_change()
    if 252 <= max_lookback and len(close) > 252:
        feat["momentum_12_1"] = close.shift(21) / close.shift(252) - 1
    if 252 <= max_lookback:
        feat["return_252d"] = close.pct_change(252)

    w52 = min(252, max_lookback)
    feat["dist_52w_high"] = close / close.rolling(w52).max() - 1

    neg_ret = ret.where(ret < 0, 0.0)
    if 252 <= max_lookback:
        feat["downside_vol_252d"] = neg_ret.rolling(252).std() * np.sqrt(252)

    if "volume" in daily.columns:
        dollar_vol = daily["volume"] * close
        amihud_daily = ret.abs() / (dollar_vol + 1e-8)
        if 252 <= max_lookback:
            feat["amihud_252d"] = amihud_daily.rolling(252).mean()

    if 252 <= max_lookback:
        rolling_max = close.rolling(252).max()
        feat["max_drawdown_252d"] = close / rolling_max - 1

    # Market context (SPY) — relative return + risk loadings only (252d).
    if spy_df is not None:
        sc = spy_df["adj_close"].reindex(daily.index, method="ffill")
        sr = sc.pct_change()
        if 252 <= max_lookback:
            feat["relative_252d"] = close.pct_change(252) - sc.pct_change(252)
            feat["corr_252d"] = ret.rolling(252).corr(sr)
            cov_252 = ret.rolling(252).cov(sr)
            var_spy = sr.rolling(252).var()
            feat["rolling_beta_252d"] = cov_252 / (var_spy + 1e-12)
            resid = ret - feat["rolling_beta_252d"] * sr
            feat["idio_vol_252d"] = resid.rolling(252).std() * np.sqrt(252)

    # ── B. Fundamental features (point-in-time) ──
    # Tier-1 prune (2026): exclude pit_pe — it is the reciprocal of pit_ep (P/E vs E/P),
    # so the two are mathematically collinear. Keep pit_ep (better-behaved at negative
    # earnings, where P/E is undefined/explosive).
    if pit_fund is not None:
        _drop_pit = {"pit_pe"}
        fund_cols = [c for c in pit_fund.columns if c.startswith("pit_") and c not in _drop_pit]
        for col in fund_cols:
            feat[col] = pit_fund[col].reindex(daily.index, method="ffill")

    # ── C. Macro features (minimal set — 2026 Tier-1 prune) ──
    # Macro features are IDENTICAL across all stocks at a given date, so they cannot help
    # cross-sectional ranking (the scanner's actual job) and only let a per-stock model
    # memorize the training period's regime — a real overfit vector against ~5 effective
    # independent yearly OOS samples. Keep a minimal, non-redundant set:
    #   macro_yield_slope  — the one yield signal that matters (recession lead indicator)
    #   macro_cpi_yoy      — inflation regime
    #   macro_gdp_growth_yoy — growth regime
    #   macro_vix          — risk appetite (added separately below)
    # Dropped: raw 10y/2y levels (collinear with slope), gdp_qoq + nonfarm_mom (overlap with
    # gdp_yoy), unemployment (lagging, overlaps gdp), and wti/natgas/copper_yoy (relevant to
    # <5% of the universe — the commodity cycle is handled separately in scan_yearly via
    # _compute_commodity_trends; as general features they were pure noise for 95% of names).
    if pit_macro is not None:
        _keep_macro = {"macro_yield_slope", "macro_cpi_yoy", "macro_gdp_growth_yoy"}
        macro_cols = [c for c in pit_macro.columns if c in _keep_macro]
        for col in macro_cols:
            feat[col] = pit_macro[col].reindex(daily.index, method="ffill")

    extra = extra or {}
    vix_df = extra.get("vix_df")
    if vix_df is not None and len(vix_df) > 20:
        vix = vix_df["vix_close"].reindex(daily.index, method="ffill")
        feat["macro_vix"] = vix
    else:
        feat["macro_vix"] = pd.Series(20.0, index=feat.index)

    # Macro × firm interaction terms removed — with only ~10 independent yearly OOS samples,
    # 11 extra interaction features added more noise than signal and degraded OOS performance.

    fundamentals_data = (extra or {}).get("fundamentals_data") or {}
    earnings_data = fundamentals_data.get("earnings", {})
    q_earnings = earnings_data.get("quarterlyEarnings", [])

    if len(q_earnings) >= 2:
        surprises = []
        for qe in q_earnings[:8]:
            reported = safe_float(qe.get("reportedEPS"))
            estimated = safe_float(qe.get("estimatedEPS"))
            if reported is not None and estimated is not None and abs(estimated) > 0.01:
                surprises.append((reported - estimated) / abs(estimated))
        if len(surprises) >= 2:
            feat["earnings_surprise_trend"] = surprises[0] - surprises[-1]
            feat["earnings_beat_rate_8q"] = sum(1 for s in surprises if s > 0) / len(surprises)
            feat["earnings_surprise_std"] = float(np.std(surprises))
            feat["latest_surprise_mag"] = surprises[0]
        else:
            for c in (
                "earnings_surprise_trend",
                "earnings_beat_rate_8q",
                "earnings_surprise_std",
                "latest_surprise_mag",
            ):
                feat[c] = np.nan
    else:
        for c in (
            "earnings_surprise_trend",
            "earnings_beat_rate_8q",
            "earnings_surprise_std",
            "latest_surprise_mag",
        ):
            feat[c] = np.nan

    overview_fd = fundamentals_data.get("overview", {})
    target_price = safe_float(overview_fd.get("AnalystTargetPrice"))
    if target_price and target_price > 0 and float(close.iloc[-1]) > 0:
        feat["analyst_target_upside"] = target_price / float(close.iloc[-1]) - 1
    else:
        feat["analyst_target_upside"] = np.nan

    # 13F-style institutional flow not available from Alpha Vantage; keep neutral (see fetch_all_data_yearly).
    feat["institutional_net_buying"] = 0.0
    feat["institutional_trend"] = 0.0
    feat["buyback_rate_yoy"] = _buyback_rate_yoy_from_shares(extra.get("shares_outstanding") or [])
    # Pruned 2026: iv_percentile_60d, volume profile (poc_distance, value_area_position,
    # hvn_support) — these are options/intraday-derived signals tuned for weekly/monthly
    # horizons. At a 252-day target their IC is near-zero and they add fitting noise.

    insider_rows = extra.get("insider") or []
    _insider_score = 0.0
    if insider_rows:
        _now = pd.Timestamp.now()
        _buy_d = _sell_d = 0.0
        for txn in insider_rows:
            try:
                txn_date = pd.Timestamp(
                    txn.get("transaction_date", txn.get("transactionDate", ""))
                )
                if (_now - txn_date).days > 30:
                    continue
                shares = abs(safe_float(txn.get("shares", txn.get("securitiesTransacted", 0))))
                price = safe_float(txn.get("share_price", txn.get("pricePerShare", 0)))
                dollars = shares * price
                acq = (
                    txn.get("acquisition_or_disposal", "") or txn.get("transactionType", "")
                ).upper()
                if acq in ("A", "P", "BUY", "PURCHASE"):
                    _buy_d += dollars
                else:
                    _sell_d += dollars
            except Exception:
                continue
        _tot = _buy_d + _sell_d
        if _tot > 0:
            _insider_score = (_buy_d - _sell_d) / _tot
    feat["insider_net_buying"] = _insider_score

    _si_pct = extra.get("short_interest_pct_float") if extra else None
    if _si_pct is not None:
        feat["short_interest_pct"] = float(np.clip(_si_pct, 0.0, 0.60))
    else:
        feat["short_interest_pct"] = 0.05  # neutral fill when unavailable

    # ── Dividend quality features ──
    _dividends = (extra or {}).get("dividends") or []
    _div_growth_5y = 0.0
    _div_streak = 0.0
    _div_yield_feat = 0.0
    if _dividends:
        try:
            _div_df = pd.DataFrame(_dividends)
            # ex_dividend_date field name varies across API versions
            _date_col = next(
                (c for c in ("ex_dividend_date", "date", "exDividendDate") if c in _div_df.columns),
                _div_df.columns[0],
            )
            _div_df["_date"] = pd.to_datetime(_div_df[_date_col], errors="coerce")
            _div_df["_amount"] = _div_df["amount"].apply(safe_float)
            _div_df = _div_df.dropna(subset=["_date", "_amount"]).sort_values("_date")
            _div_df["_year"] = _div_df["_date"].dt.year
            _annual = _div_df.groupby("_year")["_amount"].sum()
            if len(_annual) >= 2 and _annual.iloc[-1] > 0:
                # 5-year (or max available) CAGR
                _n_yr = min(5, len(_annual) - 1)
                _old = _annual.iloc[-1 - _n_yr]
                if _old > 0:
                    _div_growth_5y = float((_annual.iloc[-1] / _old) ** (1.0 / _n_yr) - 1)
                # Consecutive years without a dividend cut (5% tolerance)
                for _i in range(len(_annual) - 1, 0, -1):
                    if _annual.iloc[_i] >= _annual.iloc[_i - 1] * 0.95:
                        _div_streak += 1
                    else:
                        break
            # Trailing-12m yield proxy (annual dividends / current price)
            _last_year = _div_df[_div_df["_date"] >= (pd.Timestamp.now() - pd.Timedelta(days=365))]
            _ttm_div = float(_last_year["_amount"].sum())
            _cp = float(close.iloc[-1])
            if _cp > 0 and _ttm_div > 0:
                _div_yield_feat = _ttm_div / _cp
        except Exception:
            pass
    feat["div_growth_5y"]    = float(np.clip(_div_growth_5y, -0.50, 0.50))
    feat["div_streak_years"] = float(min(_div_streak, 50.0))
    feat["div_yield_ttm"]    = float(np.clip(_div_yield_feat, 0.0, 0.20))

    # put_call_ratio_20d pruned 2026 — contrarian sentiment is a 1-4 week signal, not 1-year.

    # ── Analyst estimate revision momentum ──
    _ee_cons = (extra or {}).get("earnings_estimates_consensus") or {}
    _ee_avg      = safe_float(_ee_cons.get("current_year_eps"))
    _ee_avg_7d   = safe_float(_ee_cons.get("eps_7d_ago"))
    _ee_avg_30d  = safe_float(_ee_cons.get("eps_30d_ago"))
    _ee_up7      = safe_float(_ee_cons.get("revisions_up_7d"))
    _ee_dn7      = safe_float(_ee_cons.get("revisions_dn_7d"))
    _ee_up30     = safe_float(_ee_cons.get("revisions_up_30d"))
    _ee_dn30     = safe_float(_ee_cons.get("revisions_dn_30d"))
    # Fractional revision drift: how much the consensus moved relative to eps magnitude
    _eps_ref = abs(_ee_avg) if _ee_avg else None
    _rev_drift_7d  = ((_ee_avg - _ee_avg_7d) / _eps_ref) if (_ee_avg and _ee_avg_7d and _eps_ref) else 0.0
    _rev_drift_30d = ((_ee_avg - _ee_avg_30d) / _eps_ref) if (_ee_avg and _ee_avg_30d and _eps_ref) else 0.0
    # Up/down ratio (positive = more upgrades, negative = more downgrades)
    _rev_ratio_7d  = ((_ee_up7 - (_ee_dn7 or 0)) / max((_ee_up7 or 0) + (_ee_dn7 or 0), 1.0)) if _ee_up7 is not None else 0.0
    _rev_ratio_30d = ((_ee_up30 - (_ee_dn30 or 0)) / max((_ee_up30 or 0) + (_ee_dn30 or 0), 1.0)) if _ee_up30 is not None else 0.0
    feat["ee_rev_drift_7d"]   = float(np.clip(_rev_drift_7d,  -0.30, 0.30))
    feat["ee_rev_drift_30d"]  = float(np.clip(_rev_drift_30d, -0.30, 0.30))
    feat["ee_rev_ratio_7d"]   = float(np.clip(_rev_ratio_7d,  -1.0,  1.0))
    feat["ee_rev_ratio_30d"]  = float(np.clip(_rev_ratio_30d, -1.0,  1.0))

    # 90-day EPS revision drift — highest-IC missing signal at 1yr horizon
    # Loaded from timestamped history cache; 0.0 if cache not yet populated (first run).
    _ee_avg_90d = safe_float(_ee_cons.get("eps_90d_ago"))
    _rev_drift_90d = (
        ((_ee_avg - _ee_avg_90d) / abs(_ee_avg)) if (_ee_avg and _ee_avg_90d and abs(_ee_avg) > 0.01) else 0.0
    )
    feat["ee_rev_drift_90d"] = float(np.clip(_rev_drift_90d, -0.50, 0.50))

    # ROE trend: YoY change in return-on-equity (identifies improving vs. deteriorating quality)
    if "pit_roe" in feat.columns:
        feat["pit_roe_trend"] = feat["pit_roe"].diff(periods=252).fillna(0.0)
    else:
        feat["pit_roe_trend"] = 0.0

    # PCA blocks (val_pca, quality_pca, risk_pca) removed — with ~10 independent yearly
    # OOS samples, 6 PCA-derived features added fitting degrees of freedom without
    # improving out-of-sample accuracy. Raw factor columns are retained as direct features.

    nullable_columns = [
        "pit_log_bm", "pit_ep", "pit_sp", "pit_gross_prof_assets", "pit_accruals",
        "pit_asset_growth", "pit_debt_to_assets", "pit_distress_prob",
        "pit_p_fcf", "pit_roe", "pit_roic", "pit_operating_margin",
        "pit_fcf_margin", "pit_d_e", "pit_interest_coverage",
        "pit_revenue_growth", "pit_earnings_growth", "pit_fcf_growth",
        # Tier-1 macro prune: only the 3 retained macro mains remain nullable.
        "macro_yield_slope", "macro_cpi_yoy", "macro_gdp_growth_yoy",
        "earnings_surprise_trend", "earnings_beat_rate_8q", "earnings_surprise_std",
        "latest_surprise_mag", "analyst_target_upside",
        "institutional_net_buying", "buyback_rate_yoy", "insider_net_buying",
        "div_growth_5y", "div_streak_years", "div_yield_ttm",
        "ee_rev_drift_7d", "ee_rev_drift_30d", "ee_rev_ratio_7d", "ee_rev_ratio_30d",
        "ee_rev_drift_90d", "pit_roe_trend",
    ]
    for col in nullable_columns:
        if col in feat.columns:
            feat[f"{col}_missing"] = feat[col].isna().astype(float)
            feat[col] = feat[col].fillna(0.0)

    # ── Target: 252-day EXCESS return (stock − SPY) — alpha-focused label ──
    stock_fwd_return = close.shift(-252) / close - 1
    if spy_df is not None and len(spy_df) > 0:
        spy_aligned = spy_df.reindex(close.index, method="ffill")
        spy_close = (
            spy_aligned["adj_close"]
            if "adj_close" in spy_aligned.columns
            else spy_aligned.iloc[:, 0]
        )
        spy_fwd_return = spy_close.shift(-252) / spy_close - 1
        feat["target"] = stock_fwd_return - spy_fwd_return
    else:
        feat["target"] = stock_fwd_return

    lower = feat["target"].quantile(0.02)
    upper = feat["target"].quantile(0.98)
    feat["target"] = feat["target"].clip(lower, upper)

    sa_lo = stock_fwd_return.quantile(0.02)
    sa_hi = stock_fwd_return.quantile(0.98)
    feat["target_absolute"] = stock_fwd_return.clip(sa_lo, sa_hi)

    return feat


# ─── ML Ensemble (Yearly) ───────────────────────────────────────────────────

_SCALED_MODELS_YEARLY = {"enet"}
_META_MIN_WEIGHT_YEARLY = 0.05


def _apply_meta_min_weight_floor_yearly(model_weights, model_names_sorted, floor=None):
    floor = _META_MIN_WEIGHT_YEARLY if floor is None else floor
    w = {nm: max(float(model_weights.get(nm, 0.0)), floor) for nm in model_names_sorted}
    t = sum(w.values())
    if t <= 0:
        u = 1.0 / len(model_names_sorted)
        return {nm: u for nm in model_names_sorted}
    return {nm: w[nm] / t for nm in model_names_sorted}


def _ensemble_pred_meta_or_weighted_yearly(
    p_matrix_row,
    model_preds_clipped,
    model_names_sorted,
    meta_model,
    model_weights,
    clip_lo=-0.80,
    clip_hi=3.0,
):
    p_matrix_row = np.atleast_2d(p_matrix_row)
    w_avg = float(
        np.clip(
            sum(
                model_weights.get(nm, 0.0) * float(model_preds_clipped.get(nm, 0.0))
                for nm in model_names_sorted
            ),
            clip_lo,
            clip_hi,
        )
    )
    if meta_model is None or not hasattr(meta_model, "predict"):
        return w_avg
    try:
        ens = float(np.clip(float(meta_model.predict(p_matrix_row)[0]), clip_lo, clip_hi))
    except Exception:
        return w_avg
    ind_sigs = []
    for nm in model_names_sorted:
        v = float(model_preds_clipped.get(nm, 0.0) or 0.0)
        if (not np.isfinite(v)) or abs(v) < 1e-12:
            continue
        ind_sigs.append(float(np.sign(v)))
    if not ind_sigs:
        return ens
    majority_sign = float(np.sign(sum(ind_sigs)))
    if majority_sign == 0.0 or np.sign(ens) == 0.0 or np.sign(ens) == majority_sign:
        return ens
    return w_avg


def _make_models_yearly(leaf=30):
    """Model factory: RF + XGB (Huber) + Elastic Net, stacked via NNLS meta-learner.

    2026 rebuild: dropped HGB (correlated with RF/XGB — inflated apparent diversification
    while feeding collinear inputs to the meta-learner) and MLP ((64,32) hidden ≈ 6k
    parameters trained on ~150 effective samples → overfits even with early stopping;
    repeatedly produced extreme outliers that the post-hoc capping had to neutralize).
    XGB switched to pseudo-Huber loss so single-year outliers (COVID, 2022, idiosyncratic
    earnings shocks) don't dominate the gradient.
    """
    return {
        "rf": RandomForestRegressor(
            n_estimators=500, max_depth=6, min_samples_leaf=leaf, random_state=42, n_jobs=-1
        ),
        "xgb": XGBRegressor(
            n_estimators=500, max_depth=4, learning_rate=0.02, subsample=0.7,
            colsample_bytree=0.5, reg_alpha=1.0, reg_lambda=2.0,
            objective="reg:pseudohubererror", huber_slope=0.05,
            min_child_weight=max(5.0, leaf / 4.0),
            random_state=42, verbosity=0,
        ),
        "enet": ElasticNet(alpha=0.10, l1_ratio=0.5, max_iter=2000, random_state=42),
    }


def _fit_yearly_model(name, model, X, y, sample_weights):
    """Weighted fit when the underlying model supports sample_weight."""
    try:
        return model.fit(X, y, sample_weight=sample_weights)
    except TypeError:
        return model.fit(X, y)


def train_ml_model_yearly(daily, spy_df=None, pit_fund=None, pit_macro=None, extra=None):
    """Train ML ensemble for yearly return prediction.
    5-window expanding walk-forward with ~252-day OOS windows."""
    IDEAL_EMBARGO_DAYS = 252
    MIN_POST_EMBARGO_ROWS = 200

    features = build_ml_features_yearly(daily, spy_df, pit_fund, pit_macro, extra=extra)
    feature_cols = [c for c in features.columns if c not in ("target", "target_absolute")]
    _special_opts = (
        "analyst_target_upside",
        "institutional_net_buying",
        "buyback_rate_yoy",
        "insider_net_buying",
        "short_interest_pct",
    )
    core_price_cols = [
        c
        for c in feature_cols
        if not c.endswith("_missing")
        and not c.startswith("pit_")
        and not c.startswith("macro_")
        and not c.startswith("earnings_")
        and c not in _special_opts
    ]
    clean = features.dropna(subset=core_price_cols + ["target"])

    if len(clean) < 100:
        return None, None, None

    # min_samples_leaf raised 2026: with 252d-overlapping forward targets, ~12 daily rows
    # share each independent observation, so leaf=40 → only ~3 truly independent samples
    # per leaf. Coarsening to len/40 forces leaves to contain ~5+ independent obs after
    # de-overlapping, sharply reducing in-sample overfitting on yearly targets.
    leaf = max(60, min(120, len(clean) // 40))
    n = len(clean)
    val_size = min(252, n // 6)
    n_windows = 5

    # ── Adaptive embargo (avoid negative train sizes on short histories) ──
    if n < MIN_POST_EMBARGO_ROWS + 60:
        EMBARGO_DAYS_EFFECTIVE = 0
        embargo_warning = f"SKIPPED_INSUFFICIENT_DATA (n={n})"
    elif n < IDEAL_EMBARGO_DAYS + MIN_POST_EMBARGO_ROWS:
        EMBARGO_DAYS_EFFECTIVE = max(21, n - MIN_POST_EMBARGO_ROWS)
        embargo_warning = f"PARTIAL_{EMBARGO_DAYS_EFFECTIVE}d (ideal {IDEAL_EMBARGO_DAYS}d)"
    else:
        EMBARGO_DAYS_EFFECTIVE = IDEAL_EMBARGO_DAYS
        embargo_warning = None

    if embargo_warning:
        print(f"    {YELLOW}⚠ Embargo: {embargo_warning}{RESET}")

    embargo_status = embargo_warning or "FULL_252D"

    if embargo_warning and "SKIPPED_INSUFFICIENT_DATA" in embargo_warning:
        print(f"    {DIM}Insufficient data for walk-forward — returning None{RESET}")
        return None, None, None

    # ── Expanding window walk-forward ──
    model_names_sorted = sorted(_make_models_yearly().keys())
    cumulative_scores = {name: [] for name in model_names_sorted}
    cumulative_rank_ics = {name: [] for name in model_names_sorted}
    all_oos_preds = []
    windows_used = 0

    for w in range(n_windows):
        val_end = n - (n_windows - 1 - w) * val_size
        val_start = val_end - val_size
        train_end = val_start - EMBARGO_DAYS_EFFECTIVE

        if train_end < 60:
            continue

        w_train = clean.iloc[:train_end]
        w_val = clean.iloc[val_start:val_end]

        X_tr = w_train[feature_cols].values
        y_tr = w_train["target"].values
        X_vl = w_val[feature_cols].values
        y_vl = w_val["target"].values

        # NaN safety net — PCA, interaction, and derived features may have NaN
        # even after core price columns are cleaned
        if np.any(np.isnan(X_tr)):
            X_tr = np.nan_to_num(X_tr, nan=0.0)
        if np.any(np.isnan(X_vl)):
            X_vl = np.nan_to_num(X_vl, nan=0.0)

        weights = np.linspace(0.33, 1.0, len(y_tr))
        w_scaler = StandardScaler()
        X_tr_s = w_scaler.fit_transform(X_tr)
        X_vl_s = w_scaler.transform(X_vl)

        w_preds = {}
        for name, model in _make_models_yearly(leaf).items():
            X_in_tr = X_tr_s if name in _SCALED_MODELS_YEARLY else X_tr
            X_in_vl = X_vl_s if name in _SCALED_MODELS_YEARLY else X_vl
            _fit_yearly_model(name, model, X_in_tr, y_tr, weights)

            vp = model.predict(X_in_vl)
            idx_subsample = list(range(0, len(vp), 21))
            if idx_subsample:
                dir_acc = np.mean(np.sign(vp[idx_subsample]) == np.sign(y_vl[idx_subsample]))
            else:
                dir_acc = np.mean(np.sign(vp) == np.sign(y_vl))
            cumulative_scores[name].append(dir_acc)
            idx_sub = list(range(0, len(vp), 21))
            if len(idx_sub) >= 5:
                r_ic, _ = spearmanr(vp[idx_sub], y_vl[idx_sub])
                r_ic = (
                    0.0
                    if r_ic is None or (isinstance(r_ic, float) and np.isnan(r_ic))
                    else float(r_ic)
                )
            else:
                r_ic = 0.0
            cumulative_rank_ics[name].append(r_ic)
            w_preds[name] = vp

        all_oos_preds.append((w_preds, y_vl))
        windows_used += 1

    if windows_used < 2:
        # Fallback single split (with embargo gap)
        val_sz = min(252, n // 5)
        val_start = n - val_sz
        tr_end = val_start - EMBARGO_DAYS_EFFECTIVE
        if tr_end < 60:
            tr_end = max(60, val_start - 126)
        w_train = clean.iloc[:tr_end]
        w_val = clean.iloc[val_start:]
        X_tr = w_train[feature_cols].values
        y_tr = w_train["target"].values
        X_vl = w_val[feature_cols].values
        y_vl = w_val["target"].values
        if np.any(np.isnan(X_tr)):
            X_tr = np.nan_to_num(X_tr, nan=0.0)
        if np.any(np.isnan(X_vl)):
            X_vl = np.nan_to_num(X_vl, nan=0.0)
        w_scaler = StandardScaler()
        X_tr_s = w_scaler.fit_transform(X_tr)
        X_vl_s = w_scaler.transform(X_vl)
        fallback_models = _make_models_yearly(leaf)
        fallback_preds = {}
        _w_fb = np.linspace(0.33, 1.0, len(y_tr))
        for name, model in fallback_models.items():
            X_in_tr = X_tr_s if name in _SCALED_MODELS_YEARLY else X_tr
            X_in_vl = X_vl_s if name in _SCALED_MODELS_YEARLY else X_vl
            _fit_yearly_model(name, model, X_in_tr, y_tr, _w_fb)
            vp = model.predict(X_in_vl)
            cumulative_scores[name] = [np.mean(np.sign(vp) == np.sign(y_vl))]
            idx_sub = list(range(0, len(vp), 21))
            if len(idx_sub) >= 5:
                r_ic, _ = spearmanr(vp[idx_sub], y_vl[idx_sub])
                r_ic = (
                    0.0
                    if r_ic is None or (isinstance(r_ic, float) and np.isnan(r_ic))
                    else float(r_ic)
                )
            else:
                r_ic = 0.0
            cumulative_rank_ics[name] = [r_ic]
            fallback_preds[name] = vp
        all_oos_preds = [(fallback_preds, y_vl)]
        windows_used = 1

    val_scores = {}
    for name, scores in cumulative_scores.items():
        if len(scores) > 0:
            val_scores[name] = float(np.mean(scores))
        else:
            val_scores[name] = 0.5

    model_rank_ics = {
        name: float(np.mean(ics)) if ics else 0.0
        for name, ics in cumulative_rank_ics.items()
    }

    meta_X_parts = []
    meta_y_parts = []
    for w_preds, y_vl in all_oos_preds:
        pred_matrix = np.column_stack(
            [np.clip(w_preds[nm], -0.80, 3.00) for nm in model_names_sorted]
        )
        meta_X_parts.append(pred_matrix)
        meta_y_parts.append(np.asarray(y_vl))
    if meta_X_parts:
        meta_X = np.vstack(meta_X_parts)
        meta_y = np.concatenate(meta_y_parts)
    else:
        meta_X = np.empty((0, len(model_names_sorted)))
        meta_y = np.array([])

    meta_model = None
    model_weights = {nm: 1.0 / len(model_names_sorted) for nm in model_names_sorted}
    if meta_X.size > 0 and len(meta_y) >= 5:
        # NNLS replaces RidgeCV+max(coefs,0)+renormalize — same answer, cleaner. Solves
        # min ‖meta_X · w − meta_y‖² subject to w ≥ 0, so weights are non-negative by
        # construction and the meta-learner cannot assign a negative "vote" to any model.
        try:
            nnls_w, _ = nnls(meta_X, meta_y)
            csum = float(nnls_w.sum())
            if csum > 0:

                class _NNLSMeta:
                    def __init__(self, weights):
                        self._w = np.asarray(weights, dtype=float)

                    def predict(self, X):
                        return np.asarray(X, dtype=float) @ self._w

                meta_model = _NNLSMeta(nnls_w)
                model_weights = {
                    model_names_sorted[i]: float(nnls_w[i] / csum)
                    for i in range(len(model_names_sorted))
                }
        except Exception:
            meta_model = None
    if meta_model is None and meta_X.size > 0 and len(meta_y) >= 1:

        class _MeanOOSMeta:
            @staticmethod
            def predict(X):
                return np.mean(X, axis=1)

        meta_model = _MeanOOSMeta()
        model_weights = {nm: 1.0 / len(model_names_sorted) for nm in model_names_sorted}

    model_weights = _apply_meta_min_weight_floor_yearly(model_weights, model_names_sorted)

    # ── Final model: training rows exclude last embargo span (label overlap) ──
    final_train_end = n - EMBARGO_DAYS_EFFECTIVE
    if final_train_end < 100:
        print(
            f"    {YELLOW}⚠ Post-embargo training rows ({final_train_end}) < 100 — "
            f"using full sample ({n} rows){RESET}"
        )
        final_train_end = n

    final_scaler = StandardScaler()
    X_all = clean[feature_cols].iloc[:final_train_end].values
    y_all = clean["target"].iloc[:final_train_end].values
    if np.any(np.isnan(X_all)):
        X_all = np.nan_to_num(X_all, nan=0.0)
    X_all_s = final_scaler.fit_transform(X_all)
    final_weights = np.linspace(0.33, 1.0, len(y_all))

    final_models = _make_models_yearly(leaf)
    for name, model in final_models.items():
        X_in = X_all_s if name in _SCALED_MODELS_YEARLY else X_all
        _fit_yearly_model(name, model, X_in, y_all, final_weights)

    # ── Predict latest — clamp features ──
    latest = features.dropna(subset=feature_cols).iloc[[-1]]
    X_latest = latest[feature_cols].values.copy()
    if np.any(np.isnan(X_latest)):
        X_latest = np.nan_to_num(X_latest, nan=0.0)
    train_df = clean[feature_cols]
    feat_lower = train_df.quantile(0.01).values
    feat_upper = train_df.quantile(0.99).values
    X_latest = np.clip(X_latest, feat_lower, feat_upper)
    X_latest_s = final_scaler.transform(X_latest)

    model_preds = {}
    model_preds_raw = {}
    for name, model in final_models.items():
        X_in = X_latest_s if name in _SCALED_MODELS_YEARLY else X_latest
        raw_p = float(model.predict(X_in)[0])
        sanity_clipped = float(np.clip(raw_p, -0.80, 3.00))
        model_preds[name] = sanity_clipped
        model_preds_raw[name] = raw_p

    P_final = np.column_stack([model_preds[nm] for nm in model_names_sorted]).reshape(1, -1)
    ensemble_pred = _ensemble_pred_meta_or_weighted_yearly(
        P_final,
        model_preds,
        model_names_sorted,
        meta_model,
        model_weights,
    )
    ensemble_pred_raw = float(
        sum(model_weights.get(nm, 0) * model_preds_raw.get(nm, 0) for nm in model_names_sorted)
    )

    # ── Post-prediction outlier reweighting (single pass: 3× median rule) ──
    # 2026 simplification: removed the ENet-vs-tree-median check that duplicated this guard.
    # NNLS meta-learner already shrinks bad models toward zero weight; this pass only fires
    # when a high-weight model produces a prediction wildly inconsistent with the panel.
    all_pv = [float(model_preds[k]) for k in model_preds]
    median_pred = float(np.median(all_pv)) if all_pv else 0.0
    for nm in list(model_weights.keys()):
        pred_val = float(model_preds.get(nm, 0))
        thr = 3.0 * abs(median_pred) + 0.10
        if abs(pred_val) > thr and model_weights.get(nm, 0) > 0.25:
            old_w = model_weights[nm]
            model_weights[nm] = 0.15
            excess = old_w - 0.15
            others = {k: v for k, v in model_weights.items() if k != nm}
            otot = sum(others.values())
            if otot > 0 and excess > 0:
                for k in others:
                    model_weights[k] += excess * (others[k] / otot)
            print(
                f"    {YELLOW}⚠ {nm} outlier prediction {pred_val*100:+.0f}% "
                f"(3× median {median_pred*100:+.0f}%) — weight capped 15%{RESET}"
            )
            ensemble_pred = sum(model_weights[x] * model_preds[x] for x in model_weights if x in model_preds)
            ensemble_pred_raw = sum(
                model_weights[x] * model_preds_raw[x] for x in model_weights if x in model_preds_raw
            )
            break

    # Mean-reversion prior on extreme trailing 12m momentum (reduces extrapolation)
    trailing_12m = 0.0
    try:
        ri = feature_cols.index("return_252d")
        trailing_12m = float(X_latest[0, ri])
    except (ValueError, IndexError):
        trailing_12m = 0.0

    mr_flag = None
    # Shrink toward 0 excess return (market-neutral prior); labels are stock − SPY
    _mr_prior = 0.0
    if trailing_12m > 2.00:
        _orig = ensemble_pred
        ensemble_pred = 0.50 * ensemble_pred + 0.50 * _mr_prior
        mr_flag = f"MEAN_REVERSION_APPLIED (12m:+{trailing_12m*100:.0f}% orig:{_orig*100:+.0f}%)"
    elif trailing_12m > 1.00:
        _orig = ensemble_pred
        ensemble_pred = 0.70 * ensemble_pred + 0.30 * _mr_prior
        mr_flag = f"MEAN_REVERSION_APPLIED (12m:+{trailing_12m*100:.0f}% orig:{_orig*100:+.0f}%)"
    elif trailing_12m > 0.50:
        _orig = ensemble_pred
        ensemble_pred = 0.85 * ensemble_pred + 0.15 * _mr_prior
        mr_flag = f"MEAN_REVERSION_APPLIED (12m:+{trailing_12m*100:.0f}% orig:{_orig*100:+.0f}%)"
    elif trailing_12m < -0.50:
        _orig = ensemble_pred
        ensemble_pred = 0.65 * ensemble_pred + 0.35 * _mr_prior
        mr_flag = f"MEAN_REVERSION_APPLIED (12m:{trailing_12m*100:+.0f}% orig:{_orig*100:+.0f}%)"

    # ── Backtest metrics (subsampled every 21d for independence) ──
    all_bt_ens = []
    all_bt_y = []
    for w_preds, y_vl in all_oos_preds:
        pred_matrix = np.column_stack(
            [np.clip(w_preds[nm], -0.80, 3.00) for nm in model_names_sorted]
        )
        bt_ens = np.zeros(pred_matrix.shape[0])
        for i in range(pred_matrix.shape[0]):
            mdict = {nm: float(pred_matrix[i, j]) for j, nm in enumerate(model_names_sorted)}
            row = pred_matrix[i : i + 1, :]
            bt_ens[i] = _ensemble_pred_meta_or_weighted_yearly(
                row,
                mdict,
                model_names_sorted,
                meta_model,
                model_weights,
            )
        idx_sub = list(range(0, len(bt_ens), 21))
        all_bt_ens.extend(bt_ens[idx_sub].tolist())
        all_bt_y.extend(y_vl[idx_sub].tolist())
    all_bt_ens = np.array(all_bt_ens)
    all_bt_y = np.array(all_bt_y)
    bt_dir_acc = np.mean(np.sign(all_bt_ens) == np.sign(all_bt_y)) if len(all_bt_ens) > 0 else 0.5
    bt_mae = np.mean(np.abs(all_bt_ens - all_bt_y)) if len(all_bt_ens) > 0 else 0.15
    n_oos = len(all_bt_y)

    # Effective OOS sample size: the backtest rows are subsampled every 21 trading days but
    # each carries a 252-day-forward (overlapping) target, so consecutive rows share ~12/13
    # of their forward window. The true number of INDEPENDENT yearly observations is far
    # lower than the raw count. Report both so downstream consumers (and the LLM judge) stop
    # over-trusting an inflated n. Overlap factor ≈ 252/21 ≈ 12.
    _OOS_OVERLAP_FACTOR = 12
    n_oos_effective = max(1, int(round(n_oos / _OOS_OVERLAP_FACTOR)))

    try:
        bt_rank_ic, _ = spearmanr(all_bt_ens, all_bt_y)
        bt_rank_ic = float(bt_rank_ic) if not np.isnan(bt_rank_ic) else 0.0
    except Exception:
        bt_rank_ic = 0.0

    # Reliability gate keyed on EFFECTIVE samples (not the inflated raw count). A high
    # dir_acc on <5 independent observations is almost certainly noise.
    confidence_penalty = 1.0
    if n_oos_effective < 1 or n_oos < 10:
        bt_dir_acc = 0.50
        confidence_penalty = 0.50
    elif bt_dir_acc > 0.70 and n_oos_effective < 6:
        print(f"    {YELLOW}⚠ Suspicious dir_acc {bt_dir_acc*100:.0f}% on only ~{n_oos_effective} "
              f"INDEPENDENT obs (raw n={n_oos}) — capping{RESET}")
        bt_dir_acc = min(bt_dir_acc, 0.62)
        confidence_penalty = 0.65

    # Feature importance from RF
    feat_importance = {}
    if "rf" in final_models:
        rf = final_models["rf"]
        imp = rf.feature_importances_
        top_idx = np.argsort(imp)[::-1][:10]
        for i in top_idx:
            feat_importance[feature_cols[i]] = float(imp[i])

    feat_imp_list = sorted(feat_importance.items(), key=lambda x: -x[1]) if feat_importance else []
    dominance_flag = None
    dominance_penalty = 1.0
    # Whitelist: factors with documented persistent predictive power at 12m horizon.
    # If one of these dominates feature importance, that's signal, not pathology — don't
    # treat it as suspicious. The penalty is for when the model leans on a single noisy
    # or non-stationary feature (typically a macro rate level).
    _factor_whitelist = {
        "pit_roe", "pit_roic", "pit_fcf_margin", "pit_operating_margin",
        "pit_earnings_growth", "pit_revenue_growth", "pit_fcf_growth",
        "pit_log_bm", "pit_ep", "pit_gross_prof_assets",
        "momentum_12_1", "return_252d", "dist_52w_high",
        "idio_vol_252d", "vol_252d", "downside_vol_252d",
        "analyst_target_upside", "ee_rev_drift_90d", "ee_rev_drift_30d",
        "earnings_beat_rate_8q",
    }
    if feat_imp_list and len(feat_imp_list) >= 3:
        top_imp = feat_imp_list[0][1]
        top_name = feat_imp_list[0][0]
        whitelisted = top_name in _factor_whitelist
        # Whitelisted features get higher dominance thresholds — they're *expected* to lead.
        if whitelisted:
            if top_imp > 0.70:
                dominance_flag = f"EXTREME_DOMINANCE_WHITELISTED ({top_name}: {top_imp*100:.0f}%)"
                dominance_penalty = 0.75
            elif top_imp > 0.55:
                dominance_flag = f"HIGH_DOMINANCE_WHITELISTED ({top_name}: {top_imp*100:.0f}%)"
                dominance_penalty = 0.92
            # < 55%: no penalty — known-good factor doing what it's supposed to.
        else:
            if top_imp > 0.60:
                dominance_flag = f"EXTREME_DOMINANCE ({top_name}: {top_imp*100:.0f}%)"
                dominance_penalty = 0.40
            elif top_imp > 0.45:
                dominance_flag = f"HIGH_DOMINANCE ({top_name}: {top_imp*100:.0f}%)"
                dominance_penalty = 0.65
            elif top_imp > 0.30:
                dominance_flag = f"MODERATE_DOMINANCE ({top_name}: {top_imp*100:.0f}%)"
                dominance_penalty = 0.85
        monotonic_risks = {
            "macro_10y_yield", "macro_2y_yield", "macro_yield_slope",
        }
        if top_name in monotonic_risks and top_imp > 0.30:
            dominance_penalty *= 0.70
            dominance_flag = (dominance_flag or "") + " [MONOTONIC_TREND_RISK]"

    # Confidence: plausibility cap. 2026 tightening — academic SOTA for 12m direction
    # prediction on equity cross-sections is ~55–58%. Raw readings >65% on ~10 OOS
    # samples are almost always sampling noise, not durable edge. Cap kicks in earlier
    # (65% vs old 75%) and compresses harder (×0.25 of excess vs old ×0.30).
    if bt_dir_acc > 0.65:
        _excess = bt_dir_acc - 0.65
        bt_dir_acc_capped = 0.65 + _excess * 0.25
        _plausibility_warning = (
            f"raw_dir_acc={bt_dir_acc*100:.0f}% → effective {bt_dir_acc_capped*100:.0f}% "
            f"(yearly plausibility limit — SOTA ≈ 55–58%)"
        )
        confidence = float(np.clip((bt_dir_acc_capped - 0.5) * 2, 0.05, 1.0)) * confidence_penalty
    else:
        bt_dir_acc_capped = bt_dir_acc
        _plausibility_warning = None
        confidence = float(np.clip((bt_dir_acc - 0.5) * 2, 0.05, 1.0)) * confidence_penalty
    confidence = float(confidence) * dominance_penalty

    quantile_preds = {"q75": 0.0, "q90": 0.0}
    try:
        if len(X_all_s) >= 200:
            for alpha_q, key in [(0.75, "q75"), (0.90, "q90")]:
                qr = GradientBoostingRegressor(
                    loss="quantile",
                    alpha=alpha_q,
                    n_estimators=200,
                    max_depth=4,
                    learning_rate=0.05,
                    subsample=0.8,
                    min_samples_leaf=20,
                    random_state=42,
                )
                qr.fit(X_all_s, y_all)
                quantile_preds[key] = float(qr.predict(X_latest_s)[0])
    except Exception as _q_exc:
        # Log instead of silently keeping the 0.0 defaults — downstream tail
        # probabilities (prob_gt_10/20/40pct) would otherwise look like real
        # neutral readings when the quantile models simply failed to fit.
        print(f"    {YELLOW}⚠ Quantile models failed ({str(_q_exc)[:60]}) — "
              f"tail probabilities use neutral defaults{RESET}")

    q75 = float(quantile_preds.get("q75", 0.0))
    q90 = float(quantile_preds.get("q90", 0.0))

    prob_positive = _sigmoid_prob(q75, threshold=0.0, scale=10.0, cap=0.95)
    prob_positive = max(prob_positive, 0.20)

    prob_gt_10pct = _sigmoid_prob(q75, threshold=0.10, scale=8.0, cap=0.80) if q75 > 0.03 else 0.0
    prob_gt_20pct = _sigmoid_prob(q90, threshold=0.20, scale=5.0, cap=0.60) if q90 > 0.05 else 0.0
    prob_gt_40pct = _sigmoid_prob(q90, threshold=0.40, scale=3.0, cap=0.40) if q90 > 0.10 else 0.0

    prob_gt_40pct = min(prob_gt_40pct, prob_gt_20pct)
    prob_gt_20pct = min(prob_gt_20pct, prob_gt_10pct)
    prob_gt_10pct = min(prob_gt_10pct, prob_positive)

    # ── Short-history magnitude clamp (2026) ─────────────────────────────────
    # On thin histories the embargo is skipped/partial and the walk-forward yields very
    # few independent OOS obs, so the ensemble freely extrapolates extreme magnitudes
    # (e.g. APP printed +104% excess on n_oos=10, embargo SKIPPED). The DIRECTION may
    # still be informative, but the MAGNITUDE is not credible — clamp it. Scales with
    # effective sample size and embargo quality. Direction (sign) is always preserved.
    _short_hist_cap = None
    _esc = str(embargo_status or "")
    if "SKIPPED" in _esc or n_oos_effective < 2:
        _short_hist_cap = 0.25
    elif "PARTIAL" in _esc or n_oos_effective < 4:
        _short_hist_cap = 0.40
    if _short_hist_cap is not None and abs(ensemble_pred) > _short_hist_cap:
        _orig_ens = ensemble_pred
        ensemble_pred = float(np.sign(ensemble_pred) * _short_hist_cap)
        print(f"    {YELLOW}⚠ Short-history magnitude clamp: excess "
              f"{_orig_ens*100:+.0f}% → {ensemble_pred*100:+.0f}% "
              f"(eff_oos~{n_oos_effective}, embargo {_esc or 'FULL'}){RESET}")

    spy_baseline = 0.08
    if spy_df is not None and len(spy_df) > 252:
        try:
            spy_adj = float(
                spy_df["adj_close"].iloc[-1] / spy_df["adj_close"].iloc[-252] - 1
            )
            spy_baseline = 0.4 * spy_adj + 0.6 * 0.08
        except Exception:
            pass
    absolute_return_pred = float(ensemble_pred + spy_baseline)
    # Bound the live point estimate. Tree/NN models EXTRAPOLATE on a single stock's 252d-forward
    # target (observed: +177% at 30% dir_acc) — the backtest clips its preds but the live output
    # was unbounded. The blend already downweights unreliable ML via confidence, but an absurd
    # headline number still misleads the reader and the LLM judge. Cap to a defensible 1yr band.
    _pre_clip = absolute_return_pred
    absolute_return_pred = float(np.clip(absolute_return_pred, -0.60, 1.00))
    if abs(_pre_clip - absolute_return_pred) > 1e-6:
        print(f"    {DIM}ML point estimate clipped {_pre_clip*100:+.0f}% → {absolute_return_pred*100:+.0f}% "
              f"(extrapolation guard){RESET}")

    prob_gt_10pct_corrected = False
    prob_positive_corrected = False
    er = float(absolute_return_pred)
    pp = float(prob_positive)
    p10 = float(prob_gt_10pct)
    if er > 0.10 and p10 < 0.05:
        prob_gt_10pct = max(0.10, pp * 0.30)
        prob_gt_10pct_corrected = True
    if er > 0.05 and pp < 0.40:
        prob_positive = max(0.45, pp)
        prob_positive_corrected = True
    if er < -0.05 and pp > 0.60:
        prob_positive = min(0.55, pp)
        prob_positive_corrected = True
    if prob_gt_10pct_corrected or prob_positive_corrected:
        prob_gt_10pct = min(prob_gt_10pct, prob_positive)
        prob_gt_20pct = min(prob_gt_20pct, prob_gt_10pct)
        prob_gt_40pct = min(prob_gt_40pct, prob_gt_20pct)

    return {
        "expected_return": absolute_return_pred,
        "expected_excess_return": float(ensemble_pred),
        "spy_baseline_assumed": float(spy_baseline),
        "expected_return_raw": float(ensemble_pred_raw + spy_baseline),
        "model_predictions": {k: float(v + spy_baseline) for k, v in model_preds.items()},
        "model_predictions_raw": {
            k: float(v + spy_baseline) for k, v in model_preds_raw.items()
        },
        "model_predictions_excess": model_preds,
        "model_weights": model_weights,
        "val_scores": val_scores,
        "model_rank_ics": model_rank_ics,
        "backtest_dir_accuracy": bt_dir_acc,
        "dir_acc_capped": float(bt_dir_acc_capped),
        "plausibility_warning": _plausibility_warning,
        "backtest_mae": bt_mae,
        "backtest_rank_ic": bt_rank_ic,
        "n_oos_samples": n_oos,
        "backtest_n_oos": n_oos,
        "n_oos_effective": n_oos_effective,
        "short_history_magnitude_cap": _short_hist_cap,
        # "Reliable" needs enough INDEPENDENT yearly observations to mean anything: 8+ (≈ eight
        # non-overlapping years) and better-than-coin-flip direction. Five points were being
        # labelled reliable.
        "dir_acc_reliable": bool(n_oos_effective >= 8 and bt_dir_acc >= 0.55),
        "confidence": confidence,
        "walk_forward_windows": windows_used,
        "feature_importance": feat_importance,
        "n_features": len(feature_cols),
        "n_training_samples": len(clean),
        "embargo_days_applied": int(EMBARGO_DAYS_EFFECTIVE),
        "embargo_status": embargo_status,
        "n_effective_training_samples": int(final_train_end),
        "prob_positive": prob_positive,
        "prob_gt_10pct": prob_gt_10pct,
        "prob_gt_10pct_corrected": prob_gt_10pct_corrected,
        "prob_gt_20pct": prob_gt_20pct,
        "prob_gt_40pct": prob_gt_40pct,
        "prob_positive_corrected": prob_positive_corrected,
        "quantile_q75": q75,
        "quantile_q90": q90,
        "mean_reversion_flag": mr_flag,
        "trailing_12m": trailing_12m,
        "feature_dominance_flag": dominance_flag,
        "feature_dominance_penalty": dominance_penalty,
    }, final_scaler, feature_cols


# ─── Cross-Sectional Panel Model ────────────────────────────────────────────






# ─── Fundamental Trajectory ML ──────────────────────────────────────────────


def _extract_quarterly_series(data):
    """Extract quarterly time series for key fundamental metrics.
    Returns dict of {metric: list_of_floats} sorted oldest→newest."""
    series = {}

    # EPS from earnings endpoint (with surprise history as auxiliary feature)
    earnings = data.get("earnings", {})
    if "quarterlyEarnings" in earnings:
        rows = [(pd.Timestamp(q["fiscalDateEnding"]),
                 safe_float(q.get("reportedEPS"), None),
                 safe_float(q.get("surprisePercentage"), None))
                for q in earnings["quarterlyEarnings"]
                if q.get("fiscalDateEnding") and safe_float(q.get("reportedEPS"), None) is not None]
        rows.sort(key=lambda x: x[0])
        if rows:
            series["eps"] = {
                "dates":     [r[0] for r in rows],
                "values":    [r[1] for r in rows],
                "surprises": [r[2] for r in rows],  # surprise % per quarter (may contain None)
            }

    # Revenue, operating income, net income, D&A from income statement
    income = data.get("income", {})
    if "quarterlyReports" in income:
        buckets = {k: [] for k in ("revenue", "operating_income", "net_income", "da")}
        for q in income["quarterlyReports"]:
            try:
                dt = pd.Timestamp(q["fiscalDateEnding"])
            except Exception:
                continue
            rev = safe_float(q.get("totalRevenue"), None)
            oi  = safe_float(q.get("operatingIncome"), None)
            ni  = safe_float(q.get("netIncome"), None)
            da  = safe_float(q.get("depreciationAndAmortization"), None)
            for name, val in [("revenue", rev), ("operating_income", oi),
                               ("net_income", ni), ("da", da)]:
                if val is not None:
                    buckets[name].append((dt, val))
        for name, rows in buckets.items():
            rows.sort(key=lambda x: x[0])
            if len(rows) >= 5:
                series[name] = {"dates": [r[0] for r in rows], "values": [r[1] for r in rows]}

    # FCF from cash flow statement
    cashflow = data.get("cashflow", {})
    if "quarterlyReports" in cashflow:
        fcf_rows = []
        for q in cashflow["quarterlyReports"]:
            try:
                dt = pd.Timestamp(q["fiscalDateEnding"])
            except Exception:
                continue
            ocf   = safe_float(q.get("operatingCashflow"), None)
            capex = safe_float(q.get("capitalExpenditures"), None)
            if ocf is not None:
                fcf = ocf - abs(capex) if capex is not None else ocf
                fcf_rows.append((dt, fcf))
        fcf_rows.sort(key=lambda x: x[0])
        if len(fcf_rows) >= 5:
            series["fcf"] = {"dates": [r[0] for r in fcf_rows], "values": [r[1] for r in fcf_rows]}

    return series


def _forecast_metric(values, n_ahead=4):
    """Train Elastic Net + XGBoost ensemble on quarterly lag features and forecast n_ahead steps.
    Returns (predictions_list, confidence) or (None, None)."""
    n = len(values)
    if n < 2:
        return None, None

    # 2-4 quarters: linear extrapolation
    if n < 5:
        slope = np.mean(np.diff(values[-min(4, n):]))
        return [values[-1] + slope * (i + 1) for i in range(n_ahead)], 0.15

    # 5-7 quarters: Holt exponential smoothing
    if n < 8:
        try:
            from statsmodels.tsa.holtwinters import ExponentialSmoothing

            model = ExponentialSmoothing(values, trend="add", seasonal=None).fit()
            forecast = model.forecast(n_ahead)
            return list(forecast), 0.30
        except Exception:
            slope = np.mean(np.diff(values[-4:]))
            return [values[-1] + slope * (i + 1) for i in range(n_ahead)], 0.20

    # Build supervised dataset: features = [lag4, lag3, lag2, lag1, qoq3, qoq2, qoq1, yoy, slope]
    X_rows, y_rows = [], []
    for i in range(4, n):
        lag = values[i - 4: i]
        qoq = []
        for j in range(1, 4):
            base = lag[j - 1]
            qoq.append((lag[j] - base) / abs(base) if base != 0 else 0.0)
        yoy_base = values[i - 5] if i >= 5 else values[0]
        yoy = (values[i - 1] - yoy_base) / abs(yoy_base) if yoy_base != 0 else 0.0
        slope = float(np.polyfit(range(4), lag, 1)[0]) if not any(np.isnan(lag)) else 0.0
        feat = list(lag) + qoq + [yoy, slope]
        X_rows.append(feat)
        y_rows.append(values[i])

    if len(X_rows) < 3:
        slope = np.mean(np.diff(values[-4:]))
        return [values[-1] + slope * (i + 1) for i in range(n_ahead)], 0.25

    X = np.nan_to_num(np.array(X_rows, dtype=float), nan=0.0)
    y = np.array(y_rows, dtype=float)

    # Exponential decay sample weights: half-life = 12 quarters (3 years).
    # Older quarters progressively down-weighted — business dynamics from 15 years
    # ago are far less predictive than the past 3 years.
    half_life = 12.0
    decay = np.exp(-np.log(2) / half_life * np.arange(len(y) - 1, -1, -1))
    sample_weights = decay / decay.sum() * len(decay)

    # OOS validation on last 2 points
    dir_acc = 0.5
    if len(X) > 4:
        X_tr, X_vl = X[:-2], X[-2:]
        y_tr, y_vl = y[:-2], y[-2:]
        w_tr = sample_weights[:-2]
        scaler_v = StandardScaler()
        r_v = ElasticNet(alpha=0.001, l1_ratio=0.5, max_iter=2000, random_state=42)
        r_v.fit(scaler_v.fit_transform(X_tr), y_tr, sample_weight=w_tr)
        pv = r_v.predict(scaler_v.transform(X_vl))
        dir_acc = float(np.mean(np.sign(pv - y_tr[-1]) == np.sign(y_vl - y_tr[-1])))

    conf = 0.25 + 0.45 * dir_acc  # 0.25 → 0.70

    # Train on full data with recency weights
    scaler = StandardScaler()
    enet = ElasticNet(alpha=0.001, l1_ratio=0.5, max_iter=2000, random_state=42)
    enet.fit(scaler.fit_transform(X), y, sample_weight=sample_weights)

    xgb_model = XGBRegressor(n_estimators=60, max_depth=3, learning_rate=0.08,
                              random_state=42, verbosity=0)
    xgb_ok = False
    try:
        xgb_model.fit(X, y, sample_weight=sample_weights)
        xgb_ok = True
    except Exception:
        pass

    # Recursive multi-step forecast
    preds = []
    history = list(values)
    for step in range(n_ahead):
        lag = history[-4:]
        qoq = []
        for j in range(1, 4):
            base = lag[j - 1]
            qoq.append((lag[j] - base) / abs(base) if base != 0 else 0.0)
        yoy_base = history[-5] if len(history) >= 5 else history[0]
        yoy = (history[-1] - yoy_base) / abs(yoy_base) if yoy_base != 0 else 0.0
        slope = float(np.polyfit(range(4), lag, 1)[0]) if not any(np.isnan(lag)) else 0.0
        feat = np.nan_to_num(np.array(lag + qoq + [yoy, slope]).reshape(1, -1), nan=0.0)
        r_p = enet.predict(scaler.transform(feat))[0]
        x_p = xgb_model.predict(feat)[0] if xgb_ok else r_p
        preds.append(float(0.5 * r_p + 0.5 * x_p))
        history.append(preds[-1])

    return preds, min(conf, 0.72)


def predict_fundamentals_ml(data):
    """
    Stage 1: Predict next 4 quarters for each key fundamental metric,
    aggregate to annual figures, and return a structured prediction dict.
    """
    quarterly = _extract_quarterly_series(data)
    if not quarterly:
        return None

    overview = data.get("overview", {})
    shares   = safe_float(overview.get("SharesOutstanding")) or \
               safe_float(overview.get("SharesFloat"))

    # Determine whether this is a cyclical (commodity-driven) company.
    # Cycle normalization thresholds are tighter for cyclicals (mean-reverting commodity EPS)
    # and looser for secular growers (genuine structural earnings improvement).
    _sec_pm = (overview.get("Sector") or "").upper()
    _ind_pm = (overview.get("Industry") or "").lower()
    _cyc_sectors = {"ENERGY", "BASIC MATERIALS", "MATERIALS"}
    _cyc_kw = {"oil", "gas", "gold", "silver", "copper", "mining", "steel",
               "aluminum", "coal", "fertilizer", "potash", "ammonia", "refin"}
    _is_cyclical_pm = (_sec_pm in _cyc_sectors or any(k in _ind_pm for k in _cyc_kw))

    pred   = {}
    raw_r  = {}

    for metric, info in quarterly.items():
        vals   = info["values"]
        preds, conf = _forecast_metric(vals, n_ahead=4)
        if preds is None:
            continue
        raw_r[metric] = {
            "quarterly_preds": preds,
            "annual_sum":  float(sum(preds)),
            "annual_mean": float(np.mean(preds)),
            "last_ttm":    float(sum(vals[-4:])) if len(vals) >= 4 else float(vals[-1] * 4),
            "confidence":  conf,
            "n_quarters":  len(vals),
        }

    if not raw_r:
        return None

    # EPS — blend ML forecast with analyst consensus + beat-rate adjustment
    if "eps" in raw_r:
        r    = raw_r["eps"]
        info = quarterly.get("eps", {})
        ml_raw_eps   = r["annual_sum"]
        ttm_eps      = r["last_ttm"]
        pred["eps_ml_raw"]       = ml_raw_eps
        pred["eps_ttm_current"]  = ttm_eps
        pred["eps_quarters"]     = r["n_quarters"]

        # Surprise history (used to calibrate the anchor)
        surprises = [s for s in info.get("surprises", []) if s is not None]
        avg_surp = 0.0
        beat_rate = 0.5
        surp_trend = 0.0
        if surprises:
            recent_surp  = surprises[-8:]
            avg_surp     = float(np.mean(recent_surp))
            beat_rate    = float(np.mean([1 if s > 0 else 0 for s in recent_surp]))
            if len(recent_surp) >= 8:
                surp_trend = float(np.mean(recent_surp[-4:])) - float(np.mean(recent_surp[:4]))
        pred["eps_avg_surprise_pct"] = avg_surp
        pred["eps_beat_rate"]        = beat_rate
        pred["eps_beat_rate_4q"]     = float(np.mean([1 if s_ > 0 else 0 for s_ in surprises[-4:]])) if surprises else None
        pred["eps_beat_n"]           = len(surprises[-8:])
        pred["eps_surprise_trend"]   = surp_trend

        # ── Analyst consensus: time-weighted composite using FY1 and FY2 estimates ──
        # The ML model predicts a rolling 12-month window. Analyst estimates are fiscal-year
        # aligned. Weight FY1 vs FY2 by how many months of each fiscal year fall inside
        # the ML's 12-month forward window, anchored to the company's fiscal year end.
        _ee = data.get("earnings_estimates_consensus") or {}
        _ee_eps       = _ee.get("current_year_eps")   # FY1 — current fiscal year consensus
        _ee_next_eps  = _ee.get("next_year_eps")       # FY2 — next fiscal year consensus
        _ee_n_analysts = int(_ee.get("analyst_count") or 0)
        if _ee_n_analysts == 0:
            # No contributors ⇒ there is no consensus. The FY figures the provider still returns are
            # not a Street number and must never anchor the forecast under a consensus label.
            _ee_eps = None; _ee_next_eps = None

        # Fiscal year end month from OVERVIEW (e.g. "December" → 12)
        _overview = data.get("overview", {})
        _fy_end_str = (_overview.get("FiscalYearEnd") or "December").strip()
        _month_map = {
            "january": 1, "february": 2, "march": 3, "april": 4,
            "may": 5, "june": 6, "july": 7, "august": 8,
            "september": 9, "october": 10, "november": 11, "december": 12,
        }
        _fy_end_month = _month_map.get(_fy_end_str.lower(), 12)
        _today = pd.Timestamp.now()
        _next_fy_close = pd.Timestamp(year=_today.year, month=_fy_end_month, day=28)
        if _next_fy_close <= _today:
            _next_fy_close = pd.Timestamp(year=_today.year + 1, month=_fy_end_month, day=28)
        _months_remaining = max(0, min(12,
            (_next_fy_close.year - _today.year) * 12
            + (_next_fy_close.month - _today.month)
        ))

        # Time-weighted composite: months remaining drives FY1 weight; complement is FY2
        _curr_w = _months_remaining / 12.0
        _next_w = 1.0 - _curr_w
        if _ee_eps is not None and _ee_next_eps is not None:
            _time_adj_anchor_raw = _curr_w * _ee_eps + _next_w * _ee_next_eps
            _consensus_source = (
                f"TIME_WEIGHTED(FY1×{_curr_w:.0%}+FY2×{_next_w:.0%},N={_ee_n_analysts})"
            )
        elif _ee_eps is not None:
            _time_adj_anchor_raw = _ee_eps
            _consensus_source = f"FY1_ONLY(N={_ee_n_analysts})"
        elif _ee_next_eps is not None:
            _time_adj_anchor_raw = _ee_next_eps
            _consensus_source = "FY2_ONLY"
        else:
            _time_adj_anchor_raw = data.get("analyst_consensus_eps")  # quarterly-sum fallback
            _consensus_source = "PROVIDER_QUARTERLY_ESTIMATES (contributor count unknown — not consensus)" if _time_adj_anchor_raw else "NONE"
        pred["eps_consensus_source"] = _consensus_source
        pred["eps_consensus_is_consensus"] = bool(_ee_n_analysts) and _consensus_source.startswith(("TIME", "FY1", "FY2"))

        # Analyst count confidence: 1 analyst → 70% trust weight; 15+ → 100%
        _analyst_conf = float(np.clip(0.70 + 0.30 * (_ee_n_analysts / 15.0), 0.70, 1.00))

        # ── Analyst growth trajectory (FY1 → FY2) vs ML implied growth ──
        analyst_eps_growth = None
        ml_implied_growth  = None
        _trajectory_conflict = False
        if (_ee_eps is not None and _ee_next_eps is not None
                and abs(_ee_eps) > 0.01 and abs(_ee_next_eps) > 0.01):
            analyst_eps_growth = (_ee_next_eps - _ee_eps) / abs(_ee_eps)
        if ttm_eps != 0:
            ml_implied_growth = (ml_raw_eps - ttm_eps) / abs(ttm_eps)
        if analyst_eps_growth is not None and ml_implied_growth is not None:
            _a_dir = int(np.sign(analyst_eps_growth))
            _m_dir = int(np.sign(ml_implied_growth))
            # Flag only when both signals are material (>10%) and point opposite directions
            _trajectory_conflict = (
                _a_dir != 0 and _m_dir != 0 and _a_dir != _m_dir
                and abs(analyst_eps_growth) > 0.10
                and abs(ml_implied_growth)  > 0.10
            )
        pred["eps_analyst_growth_fy1_fy2"] = analyst_eps_growth
        pred["eps_ml_implied_growth"]      = ml_implied_growth
        pred["eps_trajectory_conflict"]    = _trajectory_conflict

        adjusted_analyst_anchor = None
        if _time_adj_anchor_raw is not None and _time_adj_anchor_raw != 0:
            surprise_adj_factor = 1.0 + (avg_surp / 100.0) * beat_rate
            surprise_adj_factor = float(np.clip(surprise_adj_factor, 0.80, 1.30))
            adjusted_analyst_anchor = float(_time_adj_anchor_raw * surprise_adj_factor)

            pred["eps_analyst_raw"]         = float(_time_adj_anchor_raw)
            pred["eps_analyst_adjusted"]    = float(adjusted_analyst_anchor)
            pred["eps_analyst_fy1"]         = float(_ee_eps)      if _ee_eps      else None
            pred["eps_analyst_fy2"]         = float(_ee_next_eps) if _ee_next_eps else None
            pred["eps_analyst_months_rem"]  = _months_remaining
            pred["eps_surprise_adj_factor"] = surprise_adj_factor
            pred["eps_consensus_source"]    = _consensus_source
            pred["eps_analyst_count"]       = _ee_n_analysts
            pred["eps_analyst_conf"]        = _analyst_conf
        else:
            pred["eps_analyst_raw"]         = None
            pred["eps_analyst_adjusted"]    = None
            pred["eps_analyst_fy1"]         = None
            pred["eps_analyst_fy2"]         = None
            pred["eps_analyst_months_rem"]  = _months_remaining
            pred["eps_consensus_source"]    = "NONE"
            pred["eps_analyst_count"]       = 0
            pred["eps_analyst_conf"]        = 0.0

        if adjusted_analyst_anchor is not None:
            ml_anchor_divergence = (
                abs(ml_raw_eps - adjusted_analyst_anchor) / abs(adjusted_analyst_anchor)
                if adjusted_analyst_anchor != 0 else 0.0
            )

            if ml_anchor_divergence > 0.50:
                ml_weight, anchor_weight = 0.35, 0.65
                anchor_flag = "HEAVY_ANCHOR(gap>50%)"
            elif ml_anchor_divergence > 0.25:
                ml_weight, anchor_weight = 0.50, 0.50
                anchor_flag = "BALANCED_ANCHOR(gap25-50%)"
            else:
                ml_weight, anchor_weight = 0.60, 0.40
                anchor_flag = "STANDARD_ANCHOR"

            # ── Three-way divergence: is ML outside both FY1 and FY2? ──
            if (_ee_eps is not None and _ee_next_eps is not None
                    and abs(_ee_eps) > 0.01 and abs(_ee_next_eps) > 0.01):
                _lo = min(_ee_eps, _ee_next_eps) * 0.85   # 15% tolerance band
                _hi = max(_ee_eps, _ee_next_eps) * 1.15
                _ml_outside = not (_lo <= ml_raw_eps <= _hi)
                pred["eps_ml_outside_analyst_range"] = _ml_outside
                if _ml_outside and ml_anchor_divergence > 0.25:
                    anchor_weight = min(anchor_weight + 0.10, 0.80)
                    anchor_flag  += "+OUTSIDE_RANGE"
            else:
                pred["eps_ml_outside_analyst_range"] = None

            # Trajectory conflict: ML and analysts disagree on earnings direction
            if _trajectory_conflict:
                anchor_weight = min(anchor_weight + 0.10, 0.80)
                anchor_flag  += "+TRAJECTORY_CONFLICT"

            # Scale by analyst coverage: thin coverage → lean on ML
            anchor_weight = anchor_weight * _analyst_conf
            ml_weight     = 1.0 - anchor_weight

            blended_eps = ml_weight * ml_raw_eps + anchor_weight * adjusted_analyst_anchor
            pred["eps_annual"]               = float(blended_eps)
            pred["eps_ml_weight"]            = ml_weight
            pred["eps_anchor_weight"]        = anchor_weight
            pred["eps_anchor_flag"]          = anchor_flag
            pred["eps_ml_anchor_divergence"] = float(ml_anchor_divergence)
        else:
            blended_eps = ml_raw_eps
            pred["eps_annual"]               = float(blended_eps)
            pred["eps_anchor_flag"]          = "NO_ANCHOR"
            pred["eps_ml_anchor_divergence"] = None
            pred["eps_ml_outside_analyst_range"] = None

        vals_eps = info.get("values") or []
        if r["n_quarters"] >= 8 and len(vals_eps) >= 8:
            older_ttm = float(sum(vals_eps[-8:-4]))
            ttm_yoy_growth = (ttm_eps - older_ttm) / abs(older_ttm) if older_ttm != 0 else 0.0
        else:
            ttm_yoy_growth = 0.0

        pred["ttm_yoy_eps_growth"] = float(ttm_yoy_growth)

        # ── One-time charge detection ──────────────────────────────────────────
        # If the most recent quarter's EPS surprise is below -50% AND the prior
        # 3 quarters all beat, flag as likely one-time and shift anchor weight higher.
        _latest_surp = surprises[-1] if surprises else None
        _prior_surps = surprises[-4:-1] if len(surprises) >= 4 else []
        _prior_all_positive = len(_prior_surps) >= 3 and all(s > 0 for s in _prior_surps)
        _one_time_charge = (
            _latest_surp is not None
            and _latest_surp < -50.0
            and _prior_all_positive
        )
        pred["eps_one_time_charge_flag"] = _one_time_charge
        if _one_time_charge and adjusted_analyst_anchor is not None:
            # One-time miss: trust analysts more, pull anchor weight up by 15pp
            anchor_weight = min(anchor_weight + 0.15, 0.80)
            ml_weight     = 1.0 - anchor_weight
            blended_eps   = ml_weight * ml_raw_eps + anchor_weight * adjusted_analyst_anchor
            pred["eps_annual"]       = float(blended_eps)
            pred["eps_anchor_flag"]  = (pred.get("eps_anchor_flag") or "") + "+ONE_TIME_CHARGE"

        # ── Cyclically-normalized EPS ───────────────────────────────────────────
        # Anchors extreme EPS readings toward the long-run (40-quarter) median to
        # prevent cycle peaks from generating unrealistic fair value targets.
        #
        # Cyclicals (commodity/energy): tight thresholds — EPS is mean-reverting.
        # Secular growers (tech/healthcare): loose thresholds — structural improvement
        # is real; over-aggressive anchoring understates earnings quality.
        #
        # 2026 refinement: for non-cyclicals, gate the anchor on whether REVENUE is
        # also at a multi-year peak. EPS surges driven by margin expansion (LLY/GLP-1,
        # NVDA/AI) with normal revenue trajectory are structural — soften the anchor.
        # EPS surges with concurrent revenue spikes are cycle peaks — full anchor.
        _peak_threshold  = 1.50 if _is_cyclical_pm else 2.50
        _high_threshold  = 0.80 if _is_cyclical_pm else 1.50

        # Compute revenue deviation against its own 40Q median to disambiguate
        rev_vals_cycle = quarterly.get("revenue", {}).get("values") or []
        rev_deviation = None
        if len(rev_vals_cycle) >= 16:
            rev_anchor_w = rev_vals_cycle[-40:] if len(rev_vals_cycle) >= 40 else rev_vals_cycle
            rev_pos = [v for v in rev_anchor_w if v > 0]
            if rev_pos:
                rev_normalized = float(np.median(rev_pos)) * 4
                rev_ttm_local = float(sum(rev_vals_cycle[-4:])) if len(rev_vals_cycle) >= 4 else None
                if rev_normalized > 0 and rev_ttm_local:
                    rev_deviation = (rev_ttm_local - rev_normalized) / rev_normalized

        cycle_adj_label = None
        if len(vals_eps) >= 16:
            # Use up to 40 quarters for the long-run anchor; filter out negative EPS for
            # profitability-anchoring (negatives distort the mean for growth companies)
            anchor_window = vals_eps[-40:] if len(vals_eps) >= 40 else vals_eps
            pos_eps = [v for v in anchor_window if v > 0]
            if pos_eps and ttm_eps > 0:
                normalized_eps_ttm = float(np.median(pos_eps)) * 4  # median quarterly → TTM
                deviation = (ttm_eps - normalized_eps_ttm) / abs(normalized_eps_ttm) if normalized_eps_ttm != 0 else 0.0
                pred["eps_cycle_deviation"] = float(deviation)
                pred["eps_cycle_rev_deviation"] = (
                    float(rev_deviation) if rev_deviation is not None else None
                )
                pred["eps_normalized_ttm"] = float(normalized_eps_ttm)

                # Structural margin-expansion detection (non-cyclicals only):
                # EPS surged but revenue is normal → operating leverage / pricing power,
                # not a cycle peak. Soften the anchor strength accordingly.
                margin_expansion_mode = (
                    not _is_cyclical_pm
                    and rev_deviation is not None
                    and rev_deviation < 1.00   # revenue < 2× long-run median
                    and deviation > 2.00       # EPS still well above its median
                )

                if deviation > _peak_threshold:  # EPS > (1.5× cyclical | 2.5× secular) long-run median
                    if margin_expansion_mode:
                        # Structural — anchor only ~30% of the way (was 65%)
                        normalized_anchor = normalized_eps_ttm * 1.40  # allow more upside
                        anchor_w_e, blend_w_e = 0.30, 0.70
                        cycle_tag = "STRUCTURAL_MARGIN_ANCHOR"
                    else:
                        normalized_anchor = normalized_eps_ttm * 1.20
                        anchor_w_e, blend_w_e = 0.65, 0.35
                        cycle_tag = "CYCLE_PEAK_ANCHOR"
                    original_blended = blended_eps
                    blended_eps = blend_w_e * blended_eps + anchor_w_e * normalized_anchor
                    pred["eps_annual"] = float(blended_eps)
                    cycle_adj_label = (
                        f"{cycle_tag} (eps_dev:{deviation*100:+.0f}% "
                        f"rev_dev:{(rev_deviation*100) if rev_deviation is not None else 0:+.0f}% "
                        f"${original_blended:.2f}→${blended_eps:.2f})"
                    )
                elif deviation > _high_threshold:  # elevated but not peak
                    if margin_expansion_mode:
                        normalized_anchor = normalized_eps_ttm * 1.30
                        anchor_w_e, blend_w_e = 0.25, 0.75
                        cycle_tag = "STRUCTURAL_MARGIN_HIGH"
                    else:
                        normalized_anchor = normalized_eps_ttm * 1.15
                        anchor_w_e, blend_w_e = 0.45, 0.55
                        cycle_tag = "CYCLE_HIGH_ANCHOR"
                    original_blended = blended_eps
                    blended_eps = blend_w_e * blended_eps + anchor_w_e * normalized_anchor
                    pred["eps_annual"] = float(blended_eps)
                    cycle_adj_label = (
                        f"{cycle_tag} (eps_dev:{deviation*100:+.0f}% "
                        f"rev_dev:{(rev_deviation*100) if rev_deviation is not None else 0:+.0f}% "
                        f"${original_blended:.2f}→${blended_eps:.2f})"
                    )
                elif deviation < -0.60:  # current EPS < 0.4× long-run median → cycle trough
                    normalized_anchor = normalized_eps_ttm * 0.85
                    original_blended = blended_eps
                    blended_eps = 0.60 * blended_eps + 0.40 * normalized_anchor
                    pred["eps_annual"] = float(blended_eps)
                    cycle_adj_label = f"CYCLE_TROUGH_ANCHOR (dev:{deviation*100:+.0f}%)"
            else:
                pred["eps_cycle_deviation"] = None
                pred["eps_cycle_rev_deviation"] = None
                pred["eps_normalized_ttm"] = None
        else:
            pred["eps_cycle_deviation"] = None
            pred["eps_cycle_rev_deviation"] = None
            pred["eps_normalized_ttm"] = None

        # Prior YoY dampen still applies on top for extreme recent acceleration
        if cycle_adj_label:
            pred["eps_cyclical_adjustment"] = cycle_adj_label
        elif ttm_yoy_growth > 2.00:
            baseline_eps = ttm_eps * 1.10
            original_blended = blended_eps
            blended_eps = 0.40 * blended_eps + 0.60 * baseline_eps
            pred["eps_annual"] = float(blended_eps)
            pred["eps_cyclical_adjustment"] = (
                f"HEAVY_DAMPEN (TTM YoY +{ttm_yoy_growth*100:.0f}%, orig ${original_blended:.2f} → ${blended_eps:.2f})"
            )
        elif ttm_yoy_growth > 1.00:
            baseline_eps = ttm_eps * 1.15
            original_blended = blended_eps
            blended_eps = 0.55 * blended_eps + 0.45 * baseline_eps
            pred["eps_annual"] = float(blended_eps)
            pred["eps_cyclical_adjustment"] = f"MODERATE_DAMPEN (TTM YoY +{ttm_yoy_growth*100:.0f}%)"
        elif ttm_yoy_growth > 0.50:
            baseline_eps = ttm_eps * 1.20
            blended_eps = 0.70 * blended_eps + 0.30 * baseline_eps
            pred["eps_annual"] = float(blended_eps)
            pred["eps_cyclical_adjustment"] = f"LIGHT_DAMPEN (TTM YoY +{ttm_yoy_growth*100:.0f}%)"
        else:
            pred["eps_cyclical_adjustment"] = None

        # ── Consensus sanity gate ────────────────────────────────────────────────
        # A per-name model must not silently overrule a well-covered consensus: LLY's model
        # printed $20.20 (-36%) at "75% confidence" against 27 analysts at $36.71 because the
        # cyclical-peak anchor misread a structural drug launch as a commodity peak. When a
        # deep consensus exists and the model diverges by >25%, shrink hard toward consensus
        # and say so; if the divergence came from a cyclical anchor on a NON-cyclical name
        # whose consensus growth is positive, treat the anchor as a misfire and go to consensus.
        pred["eps_consensus_gate"] = None
        # typed lineage of the forecast: how much of it is the model's own (0 = it IS the consensus anchor); an
        # engine-vs-consensus comparison is published only for a forecast that is not consensus itself (relations.py)
        pred["eps_consensus_anchor"] = None
        try:
            if (data.get("restatement") and adjusted_analyst_anchor and adjusted_analyst_anchor > 0 and _ee_n_analysts >= 8):
                # Reported statements include a divested business; the per-name model was fit on
                # them. Analysts model the continuing company, so their consensus IS the forecast.
                _orig = blended_eps
                blended_eps = float(adjusted_analyst_anchor)
                pred["eps_annual"] = float(blended_eps)
                pred["eps_consensus_gate"] = (f"restatement — statements still include {data['restatement']['business']}; "
                                              f"forecast anchored to the post-divestiture consensus (N={_ee_n_analysts}): "
                                              f"${_orig:.2f} → ${blended_eps:.2f}")
                pred["eps_consensus_anchor"] = {"model_weight": 0.0, "reason": "RESTATEMENT", "n_analysts": _ee_n_analysts}
                print(f"    {YELLOW}⚠ EPS consensus gate: {pred['eps_consensus_gate']}{RESET}")
            elif adjusted_analyst_anchor and adjusted_analyst_anchor > 0 and _ee_n_analysts >= 10:
                _dev = (blended_eps - adjusted_analyst_anchor) / adjusted_analyst_anchor
                if abs(_dev) > 0.25:
                    _misfire = (cycle_adj_label and "CYCLE_PEAK" in str(cycle_adj_label)
                                and not _is_cyclical_pm and ttm_eps > 0
                                and adjusted_analyst_anchor >= ttm_eps)
                    _w_model = 0.0 if _misfire else 0.30
                    _orig = blended_eps
                    blended_eps = _w_model * blended_eps + (1 - _w_model) * adjusted_analyst_anchor
                    pred["eps_annual"] = float(blended_eps)
                    pred["eps_consensus_anchor"] = {"model_weight": _w_model, "reason": "MISFIRE" if _misfire else "DIVERGENCE",
                                                    "n_analysts": _ee_n_analysts}
                    pred["eps_consensus_gate"] = (
                        f"{'cyclical-anchor misfire on a secular grower — replaced by the' if _misfire else 'model EPS was ' + format(_dev*100, '+.0f') + '% against the'} "
                        f"coverage-weighted consensus anchor ${adjusted_analyst_anchor:.2f} (N={_ee_n_analysts}, "
                        f"FY1/FY2 time-weighted and surprise-adjusted); model forecast shrunk toward it, "
                        f"${_orig:.2f} → ${blended_eps:.2f}")
                    print(f"    {YELLOW}⚠ EPS consensus gate: {pred['eps_consensus_gate']}{RESET}")
        except Exception:
            pass

        pred["eps_growth"] = float(np.clip(
            (blended_eps - ttm_eps) / abs(ttm_eps), -1.0, 5.0
        )) if ttm_eps != 0 else 0.0

        eps_conf = r["confidence"]
        if beat_rate >= 0.75 and avg_surp > 2:
            eps_conf = min(eps_conf * 1.15, 0.75)
        elif beat_rate <= 0.35 or avg_surp < -3:
            eps_conf = eps_conf * 0.85

        # No penalty for missing analyst coverage — it's a data availability issue,
        # not a signal that the ML forecast is wrong. Uncovered stocks (small-caps,
        # international names) should be judged on ML quality alone.
        if adjusted_analyst_anchor is not None:
            divergence = pred.get("eps_ml_anchor_divergence") or 0.0
            if divergence > 0.50:
                eps_conf = eps_conf * 0.65

        pred["eps_confidence"] = float(np.clip(eps_conf, 0.05, 0.90))

    # Revenue — apply cycle normalization for cyclical/commodity companies
    if "revenue" in raw_r:
        r = raw_r["revenue"]
        rev_pred = r["annual_sum"]
        rev_ttm = r["last_ttm"]
        rev_vals = quarterly.get("revenue", {}).get("values") or []
        if len(rev_vals) >= 16 and rev_ttm > 0:
            rev_anchor_window = rev_vals[-40:] if len(rev_vals) >= 40 else rev_vals
            pos_rev = [v for v in rev_anchor_window if v > 0]
            if pos_rev:
                rev_normalized_ttm = float(np.median(pos_rev)) * 4
                rev_deviation = (rev_ttm - rev_normalized_ttm) / rev_normalized_ttm if rev_normalized_ttm != 0 else 0.0
                # Mean-reversion toward the long-run median is a COMMODITY assumption; on a
                # secular grower it manufactures a revenue collapse (LLY: -20% predicted against
                # +48% delivered and raised guidance). Apply it to cyclical names only.
                if _is_cyclical_pm and rev_deviation > 1.20:  # TTM revenue > 2.2× long-run median
                    rev_pred = 0.45 * rev_pred + 0.55 * (rev_normalized_ttm * 1.15)
                elif _is_cyclical_pm and rev_deviation > 0.60:  # elevated
                    rev_pred = 0.65 * rev_pred + 0.35 * (rev_normalized_ttm * 1.10)
        # ── Revenue consensus gate (mirror of the EPS gate) ──
        pred["rev_consensus_gate"] = None
        pred["rev_consensus_anchor"] = None
        try:
            _eec = data.get("earnings_estimates_consensus") or {}
            _r1, _r2 = _eec.get("current_year_revenue"), _eec.get("next_year_revenue")
            _nA = int(_eec.get("analyst_count") or 0)
            _rc = (0.25 * _r1 + 0.75 * _r2) if (_r1 and _r2) else (_r2 or _r1)
            if _rc and _rc > 0 and _nA >= 8 and rev_pred and data.get("restatement"):
                _orig_r = rev_pred
                rev_pred = float(_rc)
                pred["rev_consensus_gate"] = (f"restatement — reported revenue still includes {data['restatement']['business']}; "
                                              f"forecast anchored to the post-divestiture consensus (N={_nA}): "
                                              f"${_orig_r/1e9:.1f}B → ${rev_pred/1e9:.1f}B")
                pred["rev_consensus_anchor"] = {"model_weight": 0.0, "reason": "RESTATEMENT", "n_analysts": _nA}
                print(f"    {YELLOW}⚠ Revenue consensus gate: {pred['rev_consensus_gate']}{RESET}")
            elif _rc and _rc > 0 and _nA >= 10 and rev_pred:
                _rdev = (rev_pred - _rc) / _rc
                if abs(_rdev) > 0.20:
                    _orig_r = rev_pred
                    rev_pred = 0.30 * rev_pred + 0.70 * _rc
                    pred["rev_consensus_anchor"] = {"model_weight": 0.30, "reason": "DIVERGENCE", "n_analysts": _nA}
                    pred["rev_consensus_gate"] = (f"model revenue was {_rdev*100:+.0f}% against the coverage-weighted "
                                                  f"consensus anchor ${_rc/1e9:.1f}B (N={_nA}, FY1/FY2 time-weighted); "
                                                  f"model forecast shrunk toward it, ${_orig_r/1e9:.1f}B → ${rev_pred/1e9:.1f}B")
                    print(f"    {YELLOW}⚠ Revenue consensus gate: {pred['rev_consensus_gate']}{RESET}")
        except Exception:
            pass
        pred["revenue_annual"]   = rev_pred
        pred["revenue_current"]  = rev_ttm
        base = rev_ttm
        if data.get("restatement") and pred.get("rev_consensus_gate", "").startswith("restatement"):
            # like-for-like: the continuing company's current-FY consensus is the base, not the
            # consolidated TTM that still contains the divested unit (that pairing printed a
            # "-4.6% revenue decline" for a company guiding to +6-8%).
            try:
                _rb = float((data.get("earnings_estimates_consensus") or {}).get("current_year_revenue") or 0)
                if _rb > 0:
                    base = _rb; pred["revenue_current"] = _rb
                    pred["rev_base_note"] = (f"revenue growth measured against the continuing company's current-FY consensus "
                                             f"(${_rb/1e9:.1f}B), not the reported TTM (${rev_ttm/1e9:.1f}B) that still includes "
                                             f"{data['restatement']['business']}")
            except Exception:
                pass
        pred["rev_growth"]       = np.clip((rev_pred - base) / abs(base), -1.0, 2.0) if base != 0 else 0.0
        pred["rev_confidence"]   = r["confidence"]
        if shares and shares > 0:
            pred["rev_per_share"]         = rev_pred / shares
            pred["rev_per_share_current"] = rev_ttm / shares

    # Operating margin
    if "operating_income" in raw_r and "revenue" in raw_r:
        oi_pred  = raw_r["operating_income"]["annual_sum"]
        rev_pred = raw_r["revenue"]["annual_sum"]
        oi_curr  = raw_r["operating_income"]["last_ttm"]
        rev_curr = raw_r["revenue"]["last_ttm"]
        if rev_pred != 0:
            pred["op_margin_predicted"] = oi_pred / rev_pred
        if rev_curr != 0:
            pred["op_margin_current"]   = oi_curr / rev_curr
        if "op_margin_predicted" in pred and "op_margin_current" in pred:
            pred["margin_change"] = pred["op_margin_predicted"] - pred["op_margin_current"]

    # Net margin
    if "net_income" in raw_r and "revenue" in raw_r:
        ni_pred  = raw_r["net_income"]["annual_sum"]
        rev_pred = raw_r["revenue"]["annual_sum"]
        ni_curr  = raw_r["net_income"]["last_ttm"]
        rev_curr = raw_r["revenue"]["last_ttm"]
        if rev_pred != 0:
            pred["net_margin_predicted"] = ni_pred / rev_pred
        if rev_curr != 0:
            pred["net_margin_current"]   = ni_curr / rev_curr

    # FCF — cycle normalization and confidence penalty at extremes
    if "fcf" in raw_r:
        r = raw_r["fcf"]
        fcf_pred = r["annual_sum"]
        fcf_ttm = r["last_ttm"]
        fcf_conf = r["confidence"]
        fcf_vals = quarterly.get("fcf", {}).get("values") or []
        if len(fcf_vals) >= 16 and fcf_ttm > 0:
            fcf_anchor_window = fcf_vals[-40:] if len(fcf_vals) >= 40 else fcf_vals
            pos_fcf = [v for v in fcf_anchor_window if v > 0]
            if pos_fcf:
                fcf_normalized_ttm = float(np.median(pos_fcf)) * 4
                fcf_deviation = (fcf_ttm - fcf_normalized_ttm) / fcf_normalized_ttm if fcf_normalized_ttm != 0 else 0.0
                if fcf_deviation > 1.20:
                    if _is_cyclical_pm:   # mean-reversion is a commodity assumption (see revenue)
                        fcf_pred = 0.45 * fcf_pred + 0.55 * (fcf_normalized_ttm * 1.15)
                    fcf_conf *= 0.80  # wider uncertainty at cycle extremes
                elif fcf_deviation > 0.60:
                    if _is_cyclical_pm:
                        fcf_pred = 0.65 * fcf_pred + 0.35 * (fcf_normalized_ttm * 1.10)
                    fcf_conf *= 0.90
        # When deep analyst consensus has already overruled the earnings/revenue models, a
        # collapsing FCF forecast is model error, not evidence — bound the decline.
        pred["fcf_bound_note"] = None
        try:
            if (pred.get("eps_consensus_gate") or pred.get("rev_consensus_gate")) and fcf_ttm and fcf_ttm > 0:
                _floor = fcf_ttm * 0.75
                if fcf_pred < _floor:
                    pred["fcf_bound_note"] = f"FCF forecast bounded at -25% (model said {((fcf_pred/fcf_ttm)-1)*100:+.0f}%) after consensus overrode the earnings model"
                    fcf_pred = _floor
                    print(f"    {YELLOW}⚠ {pred['fcf_bound_note']}{RESET}")
        except Exception:
            pass
        pred["fcf_annual"]  = fcf_pred
        pred["fcf_current"] = fcf_ttm
        pred["fcf_confidence"] = fcf_conf
        if shares and shares > 0:
            pred["fcf_per_share"]         = fcf_pred / shares
            pred["fcf_per_share_current"] = fcf_ttm / shares

    # EPS-FCF consistency (earnings quality / capex cycle / model error)
    if "eps" in raw_r and "fcf" in raw_r:
        eps_growth = pred.get("eps_growth", 0)
        fcf_curr   = pred.get("fcf_current")
        fcf_pred   = pred.get("fcf_annual")

        if fcf_curr is not None and fcf_pred is not None and fcf_curr != 0:
            fcf_growth = (fcf_pred - fcf_curr) / abs(fcf_curr)
            pred["fcf_growth"] = float(fcf_growth)

            _gated = bool(pred.get("eps_consensus_gate"))
            if _gated:
                pred["eps_fcf_inconsistency"] = "skipped — EPS anchored to deep analyst consensus"
            elif eps_growth > 0.30 and fcf_growth < -0.15:
                pred["eps_fcf_inconsistency"] = (
                    f"HIGH (EPS +{eps_growth*100:.0f}% but FCF {fcf_growth*100:+.0f}%)"
                )
                midpoint_growth = 0.4 * eps_growth + 0.6 * fcf_growth
                midpoint_growth = max(midpoint_growth, 0.0)
                original_eps_annual = pred["eps_annual"]
                dampened_eps = pred["eps_ttm_current"] * (1 + midpoint_growth)
                pred["eps_annual"] = float(dampened_eps)
                pred["eps_growth"] = float(midpoint_growth)
                pred["eps_fcf_dampening"] = f"${original_eps_annual:.2f} → ${dampened_eps:.2f}"
                pred["eps_confidence"] = pred.get("eps_confidence", 0.5) * 0.60
            elif eps_growth > 0.50 and fcf_growth < 0:
                pred["eps_fcf_inconsistency"] = (
                    f"MODERATE (EPS +{eps_growth*100:.0f}% but FCF {fcf_growth*100:+.0f}%)"
                )
                pred["eps_confidence"] = pred.get("eps_confidence", 0.5) * 0.75
            else:
                pred["eps_fcf_inconsistency"] = None
        else:
            pred["eps_fcf_inconsistency"] = None

    if shares:
        pred["shares_outstanding"] = shares

    # ── Quality fundamentals (gross-margin trend, R&D intensity, net-debt/EBITDA) ──
    # Added 2026: at 1yr horizon, sustainable gross-margin expansion is the single
    # strongest pre-EPS leading indicator (margin compounds before it shows in NI).
    # R&D intensity is a moat proxy for tech/healthcare. Net-debt/EBITDA captures
    # balance-sheet stress that current `pit_d_e` understates (cash & EBITDA aware).
    inc_q = (data.get("income") or {}).get("quarterlyReports", [])
    bal_q = (data.get("balance") or {}).get("quarterlyReports", [])

    def _ttm_field(reports, field, start=0, n=4):
        total = 0.0
        cnt = 0
        for i in range(start, min(start + n, len(reports))):
            v = safe_float(reports[i].get(field), None)
            if v is not None:
                total += v
                cnt += 1
        return total if cnt >= max(n - 1, 1) else None

    # Gross margin: current TTM vs prior TTM (≥4 quarters earlier)
    gm_cur = gm_prev = None
    if len(inc_q) >= 4:
        gp_t = _ttm_field(inc_q, "grossProfit", 0)
        rv_t = _ttm_field(inc_q, "totalRevenue", 0)
        if gp_t is not None and rv_t and rv_t > 0:
            gm_cur = gp_t / rv_t
    if len(inc_q) >= 8:
        gp_p = _ttm_field(inc_q, "grossProfit", 4)
        rv_p = _ttm_field(inc_q, "totalRevenue", 4)
        if gp_p is not None and rv_p and rv_p > 0:
            gm_prev = gp_p / rv_p
    pred["gross_margin_ttm"]      = float(gm_cur) if gm_cur is not None else None
    pred["gross_margin_trend_yoy"] = (
        float(gm_cur - gm_prev) if (gm_cur is not None and gm_prev is not None) else None
    )

    # R&D intensity: TTM R&D / TTM Revenue (proxy for moat / reinvestment)
    if len(inc_q) >= 4:
        rd_t = _ttm_field(inc_q, "researchAndDevelopment", 0)
        rv_t = _ttm_field(inc_q, "totalRevenue", 0)
        if rd_t is not None and rv_t and rv_t > 0:
            pred["rd_intensity"] = float(rd_t / rv_t)
        else:
            pred["rd_intensity"] = None
    else:
        pred["rd_intensity"] = None

    # Net debt / EBITDA: a leverage gauge that nets out cash and scales by earnings power.
    # EBITDA proxy = operatingIncome + depreciationAndAmortization (TTM)
    net_debt_to_ebitda = None
    if bal_q and len(inc_q) >= 4:
        bs0 = bal_q[0]
        total_debt = safe_float(bs0.get("shortLongTermDebtTotal"))
        if not total_debt:
            std = safe_float(bs0.get("shortTermDebt")) or 0.0
            ltd = safe_float(bs0.get("longTermDebt")) or 0.0
            total_debt = std + ltd
        cash = (
            safe_float(bs0.get("cashAndCashEquivalentsAtCarryingValue"))
            or safe_float(bs0.get("cashAndShortTermInvestments"))
            or 0.0
        )
        oi_t = _ttm_field(inc_q, "operatingIncome", 0) or 0.0
        da_t = _ttm_field(inc_q, "depreciationAndAmortization", 0) or 0.0
        ebitda = oi_t + da_t
        if ebitda > 0:
            net_debt = (total_debt or 0.0) - cash
            net_debt_to_ebitda = net_debt / ebitda
    pred["net_debt_to_ebitda"] = (
        float(np.clip(net_debt_to_ebitda, -5.0, 15.0))
        if net_debt_to_ebitda is not None else None
    )

    confs = [raw_r[m]["confidence"] for m in raw_r]
    pred["avg_confidence"]    = float(np.mean(confs)) if confs else 0.35
    pred["metrics_predicted"] = list(raw_r.keys())
    pred["raw_results"]       = raw_r

    return pred


# ─── Sector Median Multiples ─────────────────────────────────────────────────

# Approximate market-cycle-neutral medians (blend of 10yr historical norms).
# Used for sector-relative valuation context alongside stock's own history.
_SECTOR_MEDIANS = {
    # Updated to 2026 market multiples (AI premium, rate-cut cycle in progress)
    "Technology":              {"pe": 31, "pfcf": 32, "ps": 6.5},   # was 26 — AI premium
    "Communication Services":  {"pe": 22, "pfcf": 23, "ps": 3.5},   # was 20 — streaming/ad recovery
    "Consumer Discretionary":  {"pe": 25, "pfcf": 22, "ps": 1.8},   # was 22 — e-commerce/luxury
    "Consumer Staples":        {"pe": 19, "pfcf": 21, "ps": 1.2},   # was 20 — slight compression
    "Healthcare":              {"pe": 19, "pfcf": 21, "ps": 3.5},   # was 22 — biotech derate/GLP-1
    "Financials":              {"pe": 14, "pfcf": 12, "ps": 2.7},   # was 13 — rate normalisation
    "Financial Services":      {"pe": 15, "pfcf": 13, "ps": 2.7},   # was 14
    "Industrials":             {"pe": 22, "pfcf": 24, "ps": 2.2},   # was 20 — infra spend tailwind
    "Materials":               {"pe": 15, "pfcf": 14, "ps": 1.5},
    "Energy":                  {"pe": 12, "pfcf": 10, "ps": 1.3},
    "Utilities":               {"pe": 17, "pfcf": 19, "ps": 2.1},   # was 16 — rate cut benefit
    "Real Estate":             {"pe": 38, "pfcf": 28, "ps": 7.5},   # was 40 — rate pressure eased
    "Real Estate Services":    {"pe": 20, "pfcf": 18, "ps": 1.0},   # brokers/managers (JLL, CBRE): a services P&L, not a REIT
    "Basic Materials":         {"pe": 14, "pfcf": 13, "ps": 1.4},
    "DEFAULT":                 {"pe": 20, "pfcf": 21, "ps": 2.8},   # was 18 — market re-rate
}

# D/E thresholds by sector — financial companies operate with structural leverage
_SECTOR_DE_THRESHOLDS = {
    "Financials":         {"strong": 6.0, "good": 10.0, "neutral": 15.0},
    "Financial Services": {"strong": 5.0, "good":  9.0, "neutral": 14.0},
    "Real Estate":        {"strong": 1.5, "good":  2.5, "neutral":  4.0},
    "Real Estate Services": {"strong": 0.3, "good": 0.6, "neutral": 1.2},
    "Utilities":          {"strong": 1.2, "good":  2.0, "neutral":  3.0},
    "DEFAULT":            {"strong": 0.3, "good":  0.5, "neutral":  1.0},
}


_RE_SERVICES_WORDS = ("AGENT", "MANAGER", "SERVICE", "BROKER", "OPERATIVE BUILDER", "SUBDIVIDER", "DEVELOPER", "OPERATOR")


def _is_reit_like(overview):
    """REIT-style economics (value on AFFO, REIT sector medians) vs a real-estate SERVICES firm.
    Alpha Vantage's industry string for a REIT is 'REAL ESTATE INVESTMENT TRUSTS'; a broker or
    property manager (JLL, CBRE: SIC 6531 'REAL ESTATE AGENTS & MANAGERS') shares the sector label
    but has a services P&L. Keying on the sector alone valued JLL on AFFO, skipped its story
    valuation, applied a 7.5x REIT P/S median and filled its peer table with landlords."""
    ind = str((overview or {}).get("Industry") or "").upper()
    sec = str((overview or {}).get("Sector") or "").upper()
    if "REIT" in ind or "INVESTMENT TRUST" in ind:
        return True
    if any(w in ind for w in _RE_SERVICES_WORDS):
        return False
    return sec == "REAL ESTATE"


def _sector_anchors(overview, data, reconciled=None):
    """(anchors, sector_label, lineage): the peer multiples the composite blends toward, each with its
    source, date, universe and basis.

    P/E and P/S come from Damodaran's industry datasets for the company's own industry (aggregate
    market cap ÷ trailing GAAP net income of the profitable firms; aggregate market cap ÷ revenue).
    The P/E anchor is GAAP, while the forecast EPS it multiplies is on the provider's adjusted
    definition, so it is converted with the adjusted↔GAAP bridge (P/E_adj = P/E_GAAP ÷ bridge). There is
    no sourced peer P/FCF, so that method uses the company's own history only. The legacy sector
    constants are used only when the industry dataset is unavailable, and are labelled as such."""
    legacy, sector = _get_sector_medians(overview)
    im = ((data or {}).get("damodaran_inputs") or {}).get("industry_multiples") or {}
    bridge = (((reconciled or {}).get("coherence") or {}).get("bridge") or {}).get("adjusted_to_gaap")
    out, lin = {}, {}
    if im.get("pe"):
        pe = im["pe"]
        out["pe"] = pe["value"] / bridge if bridge else pe["value"]
        lin["pe"] = dict(pe, value_as_published=pe["value"], value_used=out["pe"],
                         conversion=(f"÷ adjusted↔GAAP bridge {bridge:.4f} → adjusted-EPS basis" if bridge
                                     else "none: no adjusted↔GAAP bridge, so the GAAP anchor multiplies adjusted EPS "
                                          "(basis mismatch recorded)"),
                         basis_used="TRAILING_ADJUSTED" if bridge else "TRAILING_GAAP (applied to adjusted EPS)")
    else:
        out["pe"] = legacy["pe"]
        lin["pe"] = {"value": legacy["pe"], "value_used": legacy["pe"], "source": "Crucible sector constant (legacy, unsourced)",
                     "date": None, "universe": f"sector {sector}", "basis": "UNSTATED",
                     "conversion": "none", "note": "used only because the industry dataset is unavailable"}
    if im.get("ps"):
        out["ps"] = im["ps"]["value"]
        lin["ps"] = dict(im["ps"], value_used=out["ps"], conversion="none (revenue has one basis)")
    else:
        out["ps"] = legacy["ps"]
        lin["ps"] = {"value": legacy["ps"], "value_used": legacy["ps"], "source": "Crucible sector constant (legacy, unsourced)",
                     "date": None, "universe": f"sector {sector}", "basis": "UNSTATED",
                     "note": "used only because the industry dataset is unavailable"}
    out["pfcf"] = None
    lin["pfcf"] = {"value_used": None, "source": None,
                   "note": "no sourced peer P/FCF: the method uses the company's own history only"}
    ind = ((data or {}).get("damodaran_inputs") or {}).get("industry")
    return out, (ind or sector), lin


def _get_sector_medians(overview):
    """Return sector median multiples for a stock given its OVERVIEW dict.

    AlphaVantage returns Sector in ALL CAPS ("TECHNOLOGY", "HEALTHCARE").
    The dict keys use Title Case. Normalise with .title() before lookup.
    """
    sector_raw = (overview.get("Sector") or overview.get("GicsSector") or "").strip()
    sector = sector_raw.title()            # "TECHNOLOGY" → "Technology"
    if sector == "Real Estate" and not _is_reit_like(overview):
        sector = "Real Estate Services"
    return _SECTOR_MEDIANS.get(sector, _SECTOR_MEDIANS["DEFAULT"]), sector


# Damodaran-style corporate lifecycle → which valuation methods deserve weight (the right tool
# for the stage): a young hyper-grower is a story/revenue/DCF case (trailing P/E is meaningless);
# a mature compounder is an earnings/FCF/DCF case; a decliner is an asset/yield case.
_STAGE_WEIGHTS = {
    "young-growth":  {"P/E": 0.30, "PEG": 1.10, "P/FCF": 0.50, "P/S": 1.70, "DCF": 1.30, "Analyst Target": 1.10},
    "high-growth":   {"P/E": 0.90, "PEG": 1.40, "P/FCF": 0.90, "P/S": 1.25, "DCF": 1.15, "Analyst Target": 1.05},
    "mature-growth": {"P/E": 1.20, "PEG": 1.00, "P/FCF": 1.20, "P/S": 0.80, "DCF": 1.15, "Analyst Target": 1.00},
    "mature-stable": {"P/E": 1.20, "PEG": 0.60, "P/FCF": 1.25, "P/S": 0.60, "DCF": 1.10, "Analyst Target": 1.00},
    "decline":       {"P/E": 0.80, "PEG": 0.30, "P/FCF": 1.10, "P/S": 0.70, "DCF": 0.90, "Analyst Target": 0.80},
}


def reconciled_revenue_yoy(data):
    """Trailing revenue growth from the reconciled statements: the last four quarters over the four
    before them, on the canonical revenue concept (net revenue for a bank), after SEC reconciliation.
    None when eight quarters are not available."""
    q = ((data or {}).get("income") or {}).get("quarterlyReports") or []
    vals = [safe_float(x.get("totalRevenue")) for x in q[:8]]
    if len(vals) < 8 or any(v is None for v in vals):
        return None
    cur, prev = sum(vals[:4]), sum(vals[4:8])
    return (cur / prev - 1.0) if prev and prev > 0 else None


def consensus_growth(consensus):
    """Forward growth from sell-side consensus: FY2 over FY1, the two estimates sharing one basis and
    one perimeter (unlike FY1 over trailing, which breaks across a spin-off). None when the
    estimates are missing, non-positive or have no contributors."""
    ee = consensus or {}
    try:
        n = int(float(ee.get("analyst_count") or 0))
    except (TypeError, ValueError):
        n = 0
    if not n:
        return {"revenue": None, "eps": None, "n": 0}

    def g(a, b):
        x, y = safe_float(ee.get(a)), safe_float(ee.get(b))
        return (y / x - 1.0) if (x and y and x > 0 and y > 0) else None
    return {"revenue": g("current_year_revenue", "next_year_revenue"), "eps": g("current_year_eps", "next_year_eps"),
            "n": n, "fy1": ee.get("current_year_date"), "fy2": ee.get("next_year_date")}


def _lifecycle_stage(pred_fund, overview, is_cyclical, consensus=None, basis_out=None, trailing_rev_growth=None):
    stage, basis = _lifecycle_classify(pred_fund, overview, is_cyclical, consensus, trailing_rev_growth)
    if isinstance(basis_out, dict):
        basis_out.update(basis, stage=stage)
    return stage


def _lifecycle_classify(pred_fund, overview, is_cyclical, consensus=None, trailing_rev_growth=None):
    """Classify the Damodaran lifecycle stage from fundamentals (deterministic, no LLM).

    Damodaran stages a company on its revenue growth, profitability and reinvestment, and takes
    near-term growth from analyst estimates. FORWARD growth here is therefore sell-side consensus
    (FY2 over FY1 revenue and EPS) whenever it exists; the engine's own forecast is the fallback.

    Blends TRAILING (actual) and FORWARD (predicted) growth so a single soft forecast year
    can't mislabel a proven high-grower as 'decline'. Damodaran's decline stage means
    *sustained* shrinkage with capital return — so we require BOTH trailing and forward to
    point down before calling decline, and we stage a company UP on the stronger of its
    trailing/forward growth (a firm that has been compounding is valued on growth methods
    even if next year's forecast decelerates)."""
    cg = consensus_growth(consensus)
    rev_g = cg["revenue"] if cg["revenue"] is not None else (pred_fund.get("rev_growth", 0) or 0)
    eps_g = cg["eps"] if cg["eps"] is not None else (pred_fund.get("eps_growth", 0) or 0)
    fwd_source = ("sell-side consensus (FY2 over FY1, %d contributors)" % cg["n"]) if cg["revenue"] is not None \
        else "engine forecast (no usable consensus)"
    # Trailing EPS growth: prefer the overview's own YoY field (computed inside one currency
    # basis) — the pred_fund TTM value can mix USD analyst EPS with local-currency statement
    # EPS on foreign ADRs, printing a fake collapse that staged Kaspi as "decline".
    ttm_eps_g = safe_float(overview.get("QuarterlyEarningsGrowthYOY"))
    if ttm_eps_g is None:
        ttm_eps_g = safe_float(pred_fund.get("ttm_yoy_eps_growth"))
    # trailing revenue growth: the reconciled statements (TTM over prior TTM, canonical revenue concept)
    # when supplied; the provider's single-quarter field only as a fallback
    q_rev_g = trailing_rev_growth if trailing_rev_growth is not None else safe_float(overview.get("QuarterlyRevenueGrowthYOY"))
    npm = safe_float(overview.get("ProfitMargin"))
    div = safe_float(overview.get("DividendYield"))

    # Growth for staging = blend of trailing and forward (NOT the max: one lumpy quarter —
    # a bank's rate-driven revenue jump, a one-off — labelled JPMorgan "high-growth").
    growth = (0.5 * rev_g + 0.5 * float(np.clip(q_rev_g, -0.30, 0.80))) if q_rev_g is not None else rev_g
    # Delivered growth must not be erased by a shaky forecast: the fundamental ML is weak on
    # short-history names (Zscaler, SoundHound printed pessimistic forward growth and got
    # staged "mature-growth" at +25-80% trailing). For STAGING, let strong trailing growth
    # dominate — 75% of it is still a conservative read of what the company is doing.
    if q_rev_g is not None:
        growth = max(growth, 0.75 * float(np.clip(q_rev_g, -0.30, 0.80)))

    _B = {"trailing_revenue_growth": q_rev_g, "trailing_eps_growth": ttm_eps_g, "forward_revenue_growth": rev_g,
          "forward_eps_growth": eps_g, "forward_source": fwd_source, "staging_growth": growth,
          "net_margin": npm, "dividend_yield": div, "consensus": cg}
    # Decline only when BOTH the trailing record AND the forward forecast shrink (sustained,
    # not one soft prediction year). Missing trailing data falls back to forward-only.
    # Damodaran's decline is a SHRINKING BUSINESS. An earnings drop counts only while revenue is not
    # growing: GAAP earnings can fall on non-cash items (VST: hedge-book mark-to-market) while the
    # business grows, and that is not decline. (This replaced an LLM that could override the stage.)
    trailing_down = (q_rev_g is not None and q_rev_g < -0.02) or \
        (ttm_eps_g is not None and ttm_eps_g < -0.15 and (q_rev_g is None or q_rev_g <= 0.0))
    forward_down = rev_g < -0.02 or (eps_g < -0.15 and rev_g <= 0.0 and (q_rev_g is None or q_rev_g <= 0.0))
    if forward_down and (trailing_down or (q_rev_g is None and ttm_eps_g is None)):
        return "decline", _B
    # Maturity guard (Damodaran): firms returning meaningful cash to shareholders are past
    # the high-growth phase regardless of a strong year — high-growth firms reinvest, they
    # don't pay 2% dividends.
    _pays = bool(div and div >= 0.015)
    if growth >= 0.35 and (npm is None or npm < 0.08) and (not div or div < 0.005):
        return "young-growth", _B
    # High-growth requires the growth to be PROFITABLE growth. An unprofitable business whose
    # revenue jumps in a recovery year (Intel's foundry turnaround) is a turnaround, not a
    # high-growth compounder — value it on mature-growth inputs (shorter fade, delivered
    # economics) rather than an 18%+ growth runway.
    _profitable = (npm is not None and npm > 0.0)
    _rev_ttm = safe_float(overview.get("RevenueTTM"))
    _small = (_rev_ttm is None) or (_rev_ttm < 10e9)      # young firms are small; a $55B-revenue turnaround is not "young"
    if growth >= 0.18 and not _pays:
        if _profitable:
            return "high-growth", _B
        if _small:
            return "young-growth", _B                          # unprofitable, fast-growing, small (Rocket Lab)
        # unprofitable, fast-growing, LARGE = turnaround → mature-growth inputs (Intel)
    if growth < 0.06 and (div and div > 0.015):
        return "mature-stable", _B
    return "mature-growth", _B


def _annual_eps_series(data, n=6):
    """Reported annual EPS history (most recent first) — for mid-cycle normalization."""
    e = ((data.get("earnings") or {}).get("annualEarnings") or [])[:n]
    return [v for v in (safe_float(x.get("reportedEPS")) for x in e) if v is not None]


def _ffo_per_share(data, overview):
    """REIT AFFO/share ≈ (net income + D&A) × 0.88 / shares. FFO adds back the non-cash
    depreciation that makes REIT GAAP EPS meaningless; the 0.88 haircut approximates
    recurring maintenance capex / straight-line-rent to reach AFFO when detail is absent."""
    cf = ((data.get("cashflow") or {}).get("annualReports") or [])
    inc = ((data.get("income") or {}).get("annualReports") or [])
    if not cf or not inc:
        return None
    da = (safe_float(cf[0].get("depreciationDepletionAndAmortization"))
          or safe_float(cf[0].get("depreciationAndAmortization")))
    ni = safe_float(inc[0].get("netIncome"))
    shares = safe_float(overview.get("SharesOutstanding"))
    if da is None or ni is None or not shares or shares <= 0:
        return None
    ffo = ni + da
    if ffo <= 0:
        return None
    return (ffo * 0.88) / shares


def _book_value_per_share(data, overview):
    bs = ((data.get("balance") or {}).get("annualReports") or [])
    shares = safe_float(overview.get("SharesOutstanding"))
    if not bs or not shares or shares <= 0:
        return None
    eq = safe_float(bs[0].get("totalShareholderEquity"))
    if eq is None or eq <= 0:
        return None
    return eq / shares


def _median_roe(data, overview):
    """Median of annual net-income / shareholder-equity — a through-cycle ROE that
    ignores the current peak/trough. Falls back to the overview's TTM ROE."""
    inc = ((data.get("income") or {}).get("annualReports") or [])[:5]
    bs = ((data.get("balance") or {}).get("annualReports") or [])[:5]
    roes = []
    for i in range(min(len(inc), len(bs))):
        ni = safe_float(inc[i].get("netIncome")); eq = safe_float(bs[i].get("totalShareholderEquity"))
        if ni is not None and eq and eq > 0:
            roes.append(ni / eq)
    if roes:
        return float(np.median(roes))
    r = safe_float(overview.get("ReturnOnEquityTTM"))
    return float(r) if r and r > 0 else None


def derive_fair_value(current_price, pred_fund, data, state=None):
    """
    Stage 2: Multi-method valuation using predicted fundamentals, weighted by Damodaran lifecycle
    stage (the right valuation tool for the company's stage).
    Returns dict with fair_price, implied_return, confidence, lifecycle_stage, method breakdown.
    """
    if not pred_fund or current_price <= 0:
        return None

    overview   = data.get("overview", {})
    daily      = data.get("daily")
    earnings   = data.get("earnings", {})
    cashflow   = data.get("cashflow", {})
    # canonical inputs only (validated state + ReconciledForecastState); lineage recorded for the audit
    import pricing_inputs as _pin
    overview, pred_fund, _input_lineage = _pin.canonical_view(state, overview, pred_fund,
                                                              reconciled=(pred_fund or {}).get("_reconciled"))

    sector_upper = (overview.get("Sector") or "").upper()
    industry_lower = (overview.get("Industry") or "").lower()
    _cyclical_sectors = {"ENERGY", "BASIC MATERIALS", "REAL ESTATE", "MATERIALS"}
    _cyclical_keywords = {
        "oil", "gas", "gold", "silver", "copper", "mining",
        "reit", "mortgage", "steel", "aluminum", "lithium",
    }
    is_cyclical = sector_upper in _cyclical_sectors or any(
        k in industry_lower for k in _cyclical_keywords
    )

    if is_cyclical and pred_fund:
        pred_fund = dict(pred_fund)
        eps_growth = pred_fund.get("eps_growth", 0)
        if eps_growth and eps_growth > 0.20:
            dampened = 0.05 + 0.50 * (eps_growth - 0.05)
            pred_fund["eps_growth"] = dampened
            pred_fund["eps_cyclical_dampening"] = f"dampened {eps_growth*100:.0f}%→{dampened*100:.0f}%"
        fcf_growth = pred_fund.get("fcf_growth", 0)
        if fcf_growth and fcf_growth > 0.30:
            pred_fund["fcf_growth"] = 0.05 + 0.40 * (fcf_growth - 0.05)

    current_ps = safe_float(overview.get("PriceToSalesRatioTTM"))
    sector_meds, sector_name, _anchor_src = _sector_anchors(overview, data, (pred_fund or {}).get("_reconciled"))
    _mult_lineage = {}             # per method: the multiple, its basis and where every part of it came from
    import pricing_state as _pst
    _adj = []                      # every registered adjustment actually applied (pricing_state.ADJUSTMENTS)

    fair_values = {}   # method → (price, confidence)

    # ── Value-trap guard (acts on MAGNITUDE, not just confidence) ──────────────
    # A value trap screens cheap (low multiple → the model sees upside) but is cheap for a
    # reason: deteriorating fundamentals where the discount is deserved and the assumed
    # re-rating toward the sector median never arrives. The fv_quality_penalty below only
    # scales CONFIDENCE, so a trap still prints full optical upside. Two structural
    # defenses applied here to the fair-value MAGNITUDE:
    #   (1) DETERIORATION COMPOSITE — margin compression + predicted revenue decline +
    #       trailing earnings decline + weak ROE. 2+ signals haircut the fair price
    #       (symmetric with COMPOUNDER_FLOOR_RELIEF, which lifts magnitude for quality).
    #   (2) QUALITY-CONDITIONED RE-RATING — the sector-median blend weight shrinks with each
    #       deterioration signal, so low-quality names anchor to their OWN depressed multiple
    #       instead of being credited an unearned drift toward the sector median.
    # Predicted (forward) revenue decline is used deliberately: a cyclical at a trough has the
    # model predicting RECOVERY (rev_growth > 0), so it won't trip — distinguishing a secular
    # melting-ice-cube from a temporary cyclical low.
    _gm_trend_vt  = pred_fund.get("gross_margin_trend_yoy")
    _rev_g_vt     = pred_fund.get("rev_growth", 0) or 0
    _ttm_eps_g_vt = pred_fund.get("ttm_yoy_eps_growth")
    _roe_vt       = safe_float(overview.get("ReturnOnEquityTTM"), None)
    _vt_signals = []
    if _gm_trend_vt is not None and _gm_trend_vt < -0.005:
        _vt_signals.append("margin↓")
    if _rev_g_vt < 0.0:
        _vt_signals.append("revenue↓")
    if _ttm_eps_g_vt is not None and _ttm_eps_g_vt < 0.0:
        _vt_signals.append("earnings↓")
    if _roe_vt is not None and _roe_vt < 0.08:
        _vt_signals.append(f"ROE {_roe_vt*100:.0f}%")

    # Bank credit-quality gate: banks can't use ND/EBITDA or FCF, so the generic leverage
    # signals miss them — yet a cheap bank with a turning credit cycle is the archetypal
    # value trap, and banks are this model's favoured sector. Add credit-cycle proxies from
    # reliably-available fields: weak ROA (a healthy bank earns ~1%+; sub-0.8% signals NIM/
    # credit stress) and declining net interest income (spread compression / shrinking
    # earning assets). Provision-for-loan-losses isn't exposed by the data source, so these
    # are the best available proxies.
    if "bank" in industry_lower:
        _roa_bk = safe_float(overview.get("ReturnOnAssetsTTM"), None)
        if _roa_bk is not None and _roa_bk < 0.008:
            _vt_signals.append(f"bank ROA {_roa_bk*100:.1f}%")
        _inc_ann = (data.get("income") or {}).get("annualReports", [])
        if len(_inc_ann) >= 2:
            _nii0 = safe_float(_inc_ann[0].get("netInterestIncome"), None)
            _nii1 = safe_float(_inc_ann[1].get("netInterestIncome"), None)
            if (_nii0 is not None and _nii1 is not None and _nii1 > 0
                    and _nii0 < _nii1 * 0.97):     # NII down >3% YoY → spread compression
                _vt_signals.append("NII↓")

    _deterioration = len(_vt_signals)

    # Quality-conditioned sector-median blend weight: scale the re-rating credit down as
    # deterioration mounts (×1.0 clean → ×0.10 floor). Healthy names keep full re-rating.
    _rerate_scale = float(np.clip(1.0 - 0.30 * _deterioration, 0.10, 1.0))

    def _sec_w(base):
        """Sector-median blend weight, shrunk for deteriorating (trap-risk) names so they
        anchor to their own depressed multiple instead of an unearned sector re-rating."""
        return float(np.clip(base * _rerate_scale, 0.0, base))

    if _rerate_scale < 1.0:
        _adj.append(_pst.applied("SECTOR_RERATE_SHRINK", factor=_rerate_scale, signals=list(_vt_signals)))
    # Trap risk (2+ structural-decline signals) is flagged and lowers confidence. It no longer also
    # haircuts the composite magnitude: the peer-anchor shrink above already prices deterioration on
    # the magnitude, and pricing one risk twice on one quantity is a double count (pricing_state).
    _value_trap_flag = None
    if _deterioration >= 2:
        _value_trap_flag = f"VALUE_TRAP_RISK ({_deterioration} signals: {', '.join(_vt_signals)})"

    # ── Growth/quality adjustments (pricing_premium) ──
    # The company's own historical multiple already embeds the market's view of its quality, growth and
    # risk, so it takes only a historical-regime adjustment (change vs its own history; default 1.0). The
    # industry anchor takes an industry-RELATIVE premium: the ratio of justified multiples built from the
    # company's and the industry's growth, ROE and cost of equity (× relative net margin for P/S). The former
    # absolute premium (ROE > 20%, margin > 20%, growth > 15%, applied to the whole blend) counted the
    # company's quality twice and is gone. Cyclicals get no relative premium: their growth and ROE may sit at
    # a cycle peak.
    import pricing_premium as _ppm
    import damodaran_data as _ddp
    _dmi = data.get("damodaran_inputs") or {}
    _ind_st = (_ddp.industry_stats(_dmi.get("industry"), _dmi.get("region") or "US")
               or _ddp.industry_stats(_dmi.get("industry"), "US")) if _dmi.get("industry") else None
    _cg, _cg_src, _nc = None, None, []
    _g_ok, _trl_ok = True, True
    if state is not None:
        _f1, _f2 = state.value("consensus.eps_fy1"), state.value("consensus.eps_fy2")
        if _f1 and _f2 and _f1 > 0:
            _cg, _cg_src = _f2 / _f1 - 1.0, "sell-side consensus EPS, next fiscal year over this one (FY2 ÷ FY1)"
            _r1, _r2 = state.value("consensus.revenue_fy1"), state.value("consensus.revenue_fy2")
            _rg = (_r2 / _r1 - 1.0) if (_r1 and _r2 and _r1 > 0) else None
            # earnings expected to shrink while revenue grows: this year's EPS carries items not expected to recur,
            # so the ratio is not a growth rate
            if _rg is not None and _cg < 0 < _rg:
                _g_ok = False
                _nc.append(f"growth: consensus EPS {_cg:+.1%} vs revenue {_rg:+.1%} (non-recurring items in this year's EPS)")
        # a business separated inside the trailing window: trailing ROE and margin mix two perimeters
        if (state.historical.get("perimeter") or {}).get("event"):
            _trl_ok = False
            _nc.append("ROE and net margin: trailing figures include a separated business")
    if _cg is None and pred_fund.get("eps_growth") is not None:
        _cg, _cg_src = pred_fund.get("eps_growth"), "reconciled engine forecast EPS growth (next twelve months)"
    _rf_p = canonical_risk_free(data)
    _erp_p = float(_dmi.get("erp") or DAMODARAN_ERP)
    if is_cyclical:
        _prem = {k: {"value": 1.0, "status": "NOT_APPLIED",
                     "reason": "cyclical: current growth and ROE may sit at a cycle peak"} for k in ("pe", "ps")}
    elif _rf_p is None:
        _prem = {k: {"value": 1.0, "status": "NOT_MEASURABLE", "reason": "no risk-free rate"} for k in ("pe", "ps")}
    else:
        _prem = _ppm.industry_premiums({"growth": _cg, "growth_source": _cg_src,
                                        "roe": safe_float(overview.get("ReturnOnEquityTTM")),
                                        "beta": safe_float(_dmi.get("beta")), "net_margin": safe_float(overview.get("ProfitMargin")),
                                        "growth_comparable": _g_ok, "trailing_comparable": _trl_ok, "not_comparable": _nc},
                                       _ind_st, _rf_p, _erp_p, _rf_p)
    _regime = _ppm.historical_regime_adjustment()
    # Damodaran lifecycle stage → dynamic method weighting in the blend below.
    _stage_basis = {}
    _stage = _lifecycle_stage(pred_fund, overview, is_cyclical, consensus=data.get("earnings_estimates_consensus"),
                              basis_out=_stage_basis, trailing_rev_growth=reconciled_revenue_yoy(data))
    _stage_w = _STAGE_WEIGHTS.get(_stage, {})

    # ── Method 1: P/E with historical median ──
    eps_pred = pred_fund.get("eps_annual")
    hist_pe = None; valuation_pe = None        # exported for the target-multiple basis check
    if eps_pred and eps_pred > 0:
        # Build historical TTM EPS + price → P/E series
        hist_pe = None
        if "quarterlyEarnings" in earnings and daily is not None:
            qtrs = [(pd.Timestamp(q["fiscalDateEnding"]),
                     safe_float(q.get("reportedEPS"), None))
                    for q in earnings["quarterlyEarnings"]
                    if safe_float(q.get("reportedEPS"), None) is not None]
            qtrs.sort()
            pe_hist = []
            for i in range(3, len(qtrs)):
                ttm_eps = sum(v for _, v in qtrs[i - 3: i + 1])
                if ttm_eps > 0:
                    price_at = daily["adj_close"].asof(qtrs[i][0])
                    if price_at and not np.isnan(float(price_at)):
                        pe_val = float(price_at) / ttm_eps
                        if 5 < pe_val < 200:
                            pe_hist.append(pe_val)
            if pe_hist:
                hist_pe = float(np.median(pe_hist))

        sec_pe = sector_meds["pe"]
        _pe_parts = {"own_history": hist_pe, "own_history_basis": "median trailing P/E on the provider's adjusted EPS "
                     "(price ÷ sum of four reported quarters)", "sector_anchor": _anchor_src.get("pe")}
        _own_pe = None
        if hist_pe and 5 < hist_pe < 200:
            _own_pe, _w = hist_pe, _sec_w(0.30)          # sector weight shrinks for trap risk
        else:
            # Fall back to the current trailing multiple, put on the adjusted basis like the history
            # (price ÷ trailing adjusted EPS), else the industry anchor alone
            _eps_trl_adj = safe_float(pred_fund.get("eps_ttm_current"))
            cur_pe = (current_price / _eps_trl_adj) if (_eps_trl_adj and _eps_trl_adj > 0) else None
            _pe_parts["own_history"] = cur_pe
            _pe_parts["own_history_basis"] = "current price ÷ trailing adjusted EPS (no usable history)"
            if cur_pe and 5 < cur_pe < 200:
                _own_pe, _w = cur_pe, _sec_w(0.40)
            else:
                _w = 1.0
        _pe_w = _w
        # target = own × regime × (1 − w) + industry × industry-relative premium × w
        _ind_pe_adj = sec_pe * _prem["pe"]["value"]
        valuation_pe = ((_own_pe * _regime["value"]) * (1 - _w) if _own_pe else 0.0) + _ind_pe_adj * _w
        if _prem["pe"]["value"] != 1.0:
            _adj.append(_pst.applied("INDUSTRY_QUALITY_PREMIUM", method="P/E", component="industry_anchor",
                                     factor=_prem["pe"]["value"]))
        if _regime["value"] != 1.0 and _own_pe:
            _adj.append(_pst.applied("HISTORICAL_REGIME_ADJUSTMENT", method="P/E", component="own_history",
                                     factor=_regime["value"]))

        if valuation_pe and 5 < valuation_pe < 200:
            adj_pe = valuation_pe
            fair_pe_price = eps_pred * adj_pe
            fair_values["P/E"] = (fair_pe_price, pred_fund.get("eps_confidence", 0.40))
            _mult_lineage["P/E"] = dict(_pe_parts, sector_weight=_pe_w, industry_anchor=sec_pe,
                                        industry_quality_premium=_prem["pe"], industry_adjusted=_ind_pe_adj,
                                        historical_regime_adjustment=_regime, target_multiple=float(adj_pe),
                                        growth_quality_in_metric=False,
                                        multiple_basis="TRAILING_ADJUSTED",
                                        flow=float(eps_pred), flow_basis="NTM adjusted EPS (ReconciledForecastState.eps)",
                                        application=("trailing multiple × next-twelve-month EPS: the EPS that will be "
                                                     "trailing at the 12-month horizon, so the product is a 12-month price"),
                                        raw_price=float(fair_pe_price))

    # ── Method 2: PEG (fair P/E = EPS growth %) ──
    eps_pred   = pred_fund.get("eps_annual")
    eps_growth = pred_fund.get("eps_growth", 0)
    if eps_pred and eps_pred > 0 and eps_growth > 0.02:
        growth_pct   = eps_growth * 100                       # e.g. 15.0
        fair_peg_pe  = np.clip(growth_pct, 8.0, 45.0)        # PEG=1 rule, capped
        fair_peg_price = eps_pred * fair_peg_pe
        fair_values["PEG"] = (fair_peg_price, pred_fund.get("eps_confidence", 0.35) * 0.80)
        _mult_lineage["PEG"] = {"own_history": None, "sector_anchor": None, "sector_weight": 0.0,
                                "growth_quality_in_metric": True, "industry_quality_premium": None,
                                "historical_regime_adjustment": None,
                                "target_multiple": float(fair_peg_pe), "multiple_basis": "FORWARD_ADJUSTED",
                                "application": "PEG = 1: fair P/E equals the forecast EPS growth in percent, bounded 8–45",
                                "flow": float(eps_pred), "flow_basis": "NTM adjusted EPS (ReconciledForecastState.eps)",
                                "raw_price": float(fair_peg_price)}

    # ── Method 3: P/FCF with historical median ──
    fcf_per_share = pred_fund.get("fcf_per_share")
    shares        = pred_fund.get("shares_outstanding")
    if fcf_per_share and fcf_per_share > 0 and shares and daily is not None:
        hist_pfcf_vals = []
        if "quarterlyReports" in cashflow:
            cf_rows = []
            for q in cashflow["quarterlyReports"]:
                try:
                    dt = pd.Timestamp(q["fiscalDateEnding"])
                except Exception:
                    continue
                ocf   = safe_float(q.get("operatingCashflow"), None)
                capex = safe_float(q.get("capitalExpenditures"), None)
                if ocf is not None:
                    fcf = ocf - abs(capex) if capex is not None else ocf
                    cf_rows.append((dt, fcf))
            cf_rows.sort()
            for i in range(3, len(cf_rows)):
                ttm_fcf = sum(v for _, v in cf_rows[i - 3: i + 1])
                if ttm_fcf > 0:
                    price_at = daily["adj_close"].asof(cf_rows[i][0])
                    if price_at and not np.isnan(float(price_at)):
                        pfcf = float(price_at) / (ttm_fcf / shares)
                        if 5 < pfcf < 150:
                            hist_pfcf_vals.append(pfcf)
        # no sourced peer P/FCF exists (see _sector_anchors): the company's own history only
        # and no industry-relative premium, since there is no industry anchor to adjust (pricing_premium)
        blended_pfcf = float(np.median(hist_pfcf_vals)) if hist_pfcf_vals else None
        if blended_pfcf:
            _hist_pfcf = blended_pfcf
            blended_pfcf = blended_pfcf * _regime["value"]
        if blended_pfcf and 5 < blended_pfcf < 150:
            fair_values["P/FCF"] = (fcf_per_share * blended_pfcf,
                                    pred_fund.get("fcf_confidence", 0.38))
            _mult_lineage["P/FCF"] = {"own_history": _hist_pfcf, "own_history_basis": "median trailing price ÷ FCF per share",
                                      "sector_anchor": _anchor_src.get("pfcf"), "sector_weight": 0.0,
                                      "industry_quality_premium": None, "historical_regime_adjustment": _regime,
                                      "growth_quality_in_metric": False, "target_multiple": float(blended_pfcf),
                                      "multiple_basis": "TRAILING", "flow": float(fcf_per_share),
                                      "flow_basis": "forecast FCF per share", "raw_price": float(fcf_per_share * blended_pfcf)}

    # ── Method 4: P/S ──
    rev_per_share = pred_fund.get("rev_per_share")
    sec_ps = sector_meds["ps"]
    if rev_per_share and rev_per_share > 0:
        _own_ps = current_ps if (current_ps and 0.1 < current_ps < 50) else None
        _ps_w = _sec_w(0.35) if _own_ps else 1.0          # sector weight shrinks for trap risk
        # target = own × regime × (1 − w) + industry × industry-relative premium × w (no absolute growth multiplier)
        _ind_ps_adj = sec_ps * _prem["ps"]["value"]
        adj_ps = ((_own_ps * _regime["value"]) * (1 - _ps_w) if _own_ps else 0.0) + _ind_ps_adj * _ps_w
        if _prem["ps"]["value"] != 1.0:
            _adj.append(_pst.applied("INDUSTRY_QUALITY_PREMIUM", method="P/S", component="industry_anchor",
                                     factor=_prem["ps"]["value"]))
        fair_values["P/S"] = (rev_per_share * adj_ps,
                               pred_fund.get("rev_confidence", 0.32) * 0.75)
        _mult_lineage["P/S"] = {"own_history": current_ps,
                                "own_history_basis": "current price × shares ÷ reconciled trailing revenue",
                                "sector_anchor": _anchor_src.get("ps"), "sector_weight": _ps_w, "industry_anchor": sec_ps,
                                "industry_quality_premium": _prem["ps"], "industry_adjusted": _ind_ps_adj,
                                "historical_regime_adjustment": _regime, "growth_quality_in_metric": False,
                                "target_multiple": float(adj_ps),
                                "multiple_basis": "TRAILING", "flow": float(rev_per_share),
                                "flow_basis": "NTM revenue per share (ReconciledForecastState.revenue ÷ shares)",
                                "raw_price": float(rev_per_share * adj_ps)}

    # ── Method 5: Analyst Price Target ──
    _at_note = None
    analyst_target_ov = safe_float(overview.get("AnalystTargetPrice"))
    if analyst_target_ov and analyst_target_ov > 0:
        _at_ratio = analyst_target_ov / current_price
        if 0.5 < _at_ratio < 2.5:
            market_cap_at = safe_float(overview.get("MarketCapitalization", 0)) or 0
            _at_conf = 0.48 if market_cap_at > 50e9 else 0.40
            # Optimism haircut: consensus 12-month targets systematically overshoot. The
            # academic record (Brav & Lehavy 2003; Bradshaw, Brown & Huang 2013) puts the
            # implied return of consensus targets at roughly 2x what is realized, so only about
            # half of the implied UPSIDE is kept; a below-price target (rare, and informative)
            # is used as published. The raw target is kept in fund_detail for transparency.
            _at_impl = _at_ratio - 1.0
            _at_adj = current_price * (1.0 + (0.55 * _at_impl if _at_impl > 0 else _at_impl))
            fair_values["Analyst Target"] = (float(_at_adj), _at_conf)
            _mult_lineage["Analyst Target"] = {"flow": float(analyst_target_ov), "flow_basis": "mean sell-side 12-month target",
                                               "source": "consensus.target_price", "raw_price": float(_at_adj)}
            if _at_impl > 0:
                _adj.append(_pst.applied("ANALYST_TARGET_OPTIMISM", method="Analyst Target", factor=0.55,
                                         raw=float(analyst_target_ov), adjusted=float(_at_adj)))
            if _at_impl > 0.02:
                _at_note = (f"ANALYST_TARGET_HAIRCUT: consensus target ${analyst_target_ov:,.2f} ({_at_impl*100:+.0f}%) "
                            f"used at ${_at_adj:,.2f} ({(_at_adj/current_price-1)*100:+.0f}%) — "
                            f"sell-side targets realize about half their implied upside")

    # ── Method 6: Damodaran-style 2-stage DCF ─────────────────────────────────────
    # Upgraded (2026-08) to follow Damodaran's published discipline rather than a generic
    # FCF perpetuity (researched against his NYU pages / lifecycle book / 2025 lectures):
    #   * discount rate: CAPM with the LIVE 10Y riskfree + his IMPLIED equity risk premium
    #     (4.28% as of Aug 2026 — he republishes monthly; update _DAMODARAN_ERP with it),
    #     with BETA FADING to 1.0 by maturity (his cost-of-capital fade), not a flat 5.5% ERP
    #   * growth must be PAID FOR: earnings-based flows retain only payout = 1 - g/ROE
    #     (his fundamental-growth equation) — growing flows at 30%/yr while distributing
    #     100% of them is the exact "growth is free" error his terminal-value lecture flags
    #   * terminal growth ≤ the riskfree rate, terminal payout = 1 - g/stable-ROE
    #   * young-growth names carry a failure-probability haircut (going-concern adjustment)
    # Flows: predicted forward EPS preferred (reinvestment link is exact for earnings);
    # FCF/share fallback (already net of current reinvestment, so no payout factor —
    # instead growth is capped at the ROE-sustainable rate to avoid unfunded growth).
    _di = data.get("damodaran_inputs") or {}
    # his implied ERP for the month (+ country risk premium of the country of operations); the documented
    # constant is used only when the dataset is unavailable, and the source is recorded either way
    _DAMODARAN_ERP = float(_di.get("erp") or DAMODARAN_ERP)
    _dcf_meta = None
    try:
        _t10_raw = data.get("treasury_10y", [])
        _rf = float(_t10_raw[0]["value"]) / 100 if _t10_raw else None
        data["rf_source"] = "Alpha Vantage TREASURY_YIELD 10y (daily)" if _t10_raw else "UNAVAILABLE"
    except (TypeError, KeyError, IndexError, ValueError):
        _rf = None
        data["rf_source"] = "UNAVAILABLE"
    # Regression betas are noisy (Damodaran uses bottom-up betas for this reason) —
    # start from the bottom-up industry beta and fade it to exactly 1.0 in stable growth
    # (his convention: a mature firm is discounted like the median US company).
    try:
        from story_dcf import damodaran_betas as _dbetas
        _beta0, _beta_stable = _dbetas(safe_float(overview.get("Beta")), overview.get("Sector"), overview.get("Industry"),
                                       bottom_up=_di.get("beta"))
    except Exception:  # noqa: BLE001
        _beta0 = float(np.clip(safe_float(overview.get("Beta")) or 1.0, 0.5, 2.0)); _beta_stable = 1.0
    _roe_d = safe_float(overview.get("ReturnOnEquityTTM"))
    _roe_d = float(np.clip(_roe_d if _roe_d and _roe_d > 0 else 0.14, 0.06, 0.40))
    # first-stage growth cap of the mechanical cross-check: wider only for a company that is better than its industry
    _g1_cap = 0.25 if is_cyclical else float(np.clip(0.25 * float(_prem["pe"].get("raw_ratio") or _prem["pe"]["value"]), 0.25, 0.40))
    _N = {"young-growth": 8, "high-growth": 6, "mature-growth": 5,
          "mature-stable": 4, "decline": 3}.get(_stage, 5)
    _g_term = float(min(0.04, _rf))                       # stable growth ≤ riskfree (his rule; his default is g = rf, capped 4%)
    _roe_stable = float(np.clip(_roe_d, 0.08, 0.20))      # excess returns compress at maturity

    # ── Normalized intrinsic base — GAAP EPS misrepresents economics for some structures.
    #    REIT: non-cash depreciation guts EPS → value on AFFO/share. Insurer/bank: cyclical
    #    peak EPS + AOCI/reserve swings → value on book value × justified P/B (through-cycle
    #    ROE), a residual-income anchor rather than a growth DCF on peak earnings. Cyclical
    #    in a non-growth stage: point EPS is a commodity snapshot → mid-cycle (median) EPS.
    _secU = str(overview.get("Sector") or "").upper(); _indU = str(overview.get("Industry") or "").upper()
    _fw_sel = (data or {}).get("valuation_framework") or {}
    _is_reit_v = (_fw_sel.get("mechanical_basis") == "affo") if _fw_sel else _is_reit_like(overview)
    _is_fin_v = (_fw_sel.get("mechanical_basis") == "book_roe") if _fw_sel else ((_secU == "FINANCIAL SERVICES") and any(
        k in _indU for k in ("BANK", "INSURANCE", "CAPITAL MARKETS", "ASSET MANAGEMENT", "CREDIT")))
    _norm_note = None; _direct_value = None; _norm_flow = None; _norm_basis = None
    if _is_reit_v:
        _affo = _ffo_per_share(data, overview)
        if _affo and _affo > 0:
            _norm_flow, _norm_basis = _affo, "affo"
            _norm_note = "REIT — valued on AFFO/share (GAAP EPS understates by non-cash depreciation)"
    elif _is_fin_v:
        _bvps = _book_value_per_share(data, overview)
        if _bvps and _bvps > 0:
            _roe_med = _median_roe(data, overview)
            _roe_sus = float(np.clip(_roe_med, 0.08, 0.18)) if _roe_med is not None else None
        if _bvps and _bvps > 0 and _roe_sus is None:
            _norm_note = "bank/insurer — through-cycle ROE unavailable; book-value model withheld"
        if _bvps and _bvps > 0 and _roe_sus is not None and _rf is not None:
            _ke_fin = float(np.clip(_rf + _beta_stable * _DAMODARAN_ERP, 0.065, 0.16))
            _jpb = float(np.clip((_roe_sus - _g_term) / max(_ke_fin - _g_term, 0.02), 0.6, 3.0))
            _direct_value = _bvps * _jpb
            _norm_note = (f"insurer/bank — book value ${_bvps:,.1f}/sh × justified P/B {_jpb:.2f} "
                          f"(through-cycle ROE {_roe_sus*100:.0f}%, ke {_ke_fin*100:.1f}%, g {_g_term*100:.1f}%)")
            if 0.20 * current_price < _direct_value < 5.0 * current_price:
                _dcf_meta = {"basis": "book_roe", "rf": round(_rf, 4), "erp": _DAMODARAN_ERP,
                             "beta": round(_beta0, 2), "beta_stable": round(_beta_stable, 2),
                             "ke_initial": round(_ke_fin, 4), "roe": round(_roe_sus, 3),
                             "bvps": round(_bvps, 2), "justified_pb": round(_jpb, 2),
                             "g_terminal": round(_g_term, 4), "high_growth_years": _N,
                             "survival": 1.0, "norm_note": _norm_note}
    elif is_cyclical and _stage in ("mature-stable", "decline"):
        _eps_ser = _annual_eps_series(data)
        if len(_eps_ser) >= 3:
            _mid = float(np.median(_eps_ser))
            if _mid and _mid > 0:
                _norm_flow, _norm_basis = _mid, "eps"
                _norm_note = (f"cyclical — mid-cycle EPS ${_mid:.2f} (median of {len(_eps_ser)}y) "
                              f"vs point ${(pred_fund.get('eps_annual') or 0):.2f}")

    if _rf is None:
        # the risk-free rate is a market input, not a constant: without it there is no cost of equity
        # and therefore no per-share value — the engine reports UNAVAILABLE instead of using a default
        _direct_value = None; _norm_basis = None; _norm_note = (_norm_note or "") + " | per-share value withheld: risk-free rate unavailable"
    if _direct_value is not None and 0.20 * current_price < _direct_value < 5.0 * current_price:
        # book value × justified P/B — a direct residual-income value; skip the flow DCF.
        fair_values["DCF"] = (float(_direct_value), 0.55)

    _eps_flow = (_norm_flow if (_norm_basis == "eps" and _norm_flow) else pred_fund.get("eps_annual"))
    _fcf_flow = (_norm_flow if (_norm_basis == "affo" and _norm_flow) else pred_fund.get("fcf_per_share"))
    _basis = (_norm_basis if _norm_basis else
              ("eps" if (_eps_flow and _eps_flow > 0) else ("fcf" if (_fcf_flow and _fcf_flow > 0) else None)))
    if _rf is None:
        _basis = None
    if _direct_value is None and _basis is not None:
        # Growth input: BLEND forward (ML-predicted) with trailing (delivered) growth —
        # a single soft forecast year must not value a compounding business as shrinking
        # (same guard as _lifecycle_stage: NVDA's forward eps_growth printed negative and
        # the DCF collapsed to -78%). Damodaran anchors near-term growth on the revenue
        # story/delivered economics, not one point forecast.
        # near-term growth: analyst estimates first (his convention), the engine's forecast otherwise
        _cg_eps = (_stage_basis.get("consensus") or {}).get("eps")
        _g_fwd = _cg_eps if (_basis == "eps" and _cg_eps is not None) else pred_fund.get("eps_growth" if _basis == "eps" else "fcf_growth")
        if _g_fwd is None:
            _g_fwd = pred_fund.get("eps_growth")
        if _g_fwd is None:
            _basis = None                              # no growth input → no mechanical value (never a default 8%)
            _norm_note = (_norm_note or "") + " | mechanical value withheld: growth input unavailable"
    if _direct_value is None and _basis is not None:
        _g_fwd = float(_g_fwd)
        _g_trl = safe_float(pred_fund.get("ttm_yoy_eps_growth"))
        if _g_trl is None:
            _g_trl = reconciled_revenue_yoy(data)
            if _g_trl is None:
                _g_trl = safe_float(overview.get("QuarterlyRevenueGrowthYOY"))
        _g_blend = (0.5 * _g_fwd + 0.5 * float(np.clip(_g_trl, -0.20, 0.60))) \
                   if _g_trl is not None else _g_fwd
        if _basis == "eps":
            _flow0 = float(_eps_flow)
            _g1 = float(np.clip(_g_blend, -0.05, _g1_cap))
        elif _basis == "affo":
            # AFFO is a cash measure already net of maintenance capex; its growth comes from
            # rent + development, NOT retained-earnings ROE, so it is not ROE-capped.
            _flow0 = float(_fcf_flow)
            _g1 = float(np.clip(_g_blend, -0.05, _g1_cap))
        else:
            _flow0 = float(_fcf_flow)
            # FCF is already net of reinvestment → growth can't exceed what full
            # reinvestment of earnings at ROE could fund.
            _g1 = float(np.clip(_g_blend, -0.05, min(_g1_cap, _roe_d)))
        _survival = 0.90 if _stage == "young-growth" else 1.0   # going-concern haircut
        from valuation_engines import two_stage_per_share as _tsps
        pv, _tsm = _tsps(_flow0, _g1, _g_term, _N, _rf, _DAMODARAN_ERP, _beta0, _beta_stable, _roe_d, _roe_stable, _basis, _survival)
        if 0.20 * current_price < pv < 5.0 * current_price:
            dcf_conf = (pred_fund.get("eps_confidence", 0.40) if _basis == "eps"
                        else pred_fund.get("fcf_confidence", 0.35)) * 0.90
            fair_values["DCF"] = (float(pv), float(dcf_conf))
            _dcf_meta = {"basis": _basis, "rf": round(_rf, 4), "erp": _DAMODARAN_ERP,
                         "beta": round(_beta0, 2), "beta_stable": round(_beta_stable, 2), "ke_initial": round(float(np.clip(_rf + _beta0 * _DAMODARAN_ERP, 0.065, 0.16)), 4),
                         "g1": round(_g1, 4), "g1_forward": round(float(_g_fwd), 4),
                         "g1_trailing": (round(float(_g_trl), 4) if _g_trl is not None else None),
                         "g_terminal": round(_g_term, 4),
                         "high_growth_years": _N, "roe": round(_roe_d, 3),
                         "survival": _survival, "norm_note": _norm_note,
                         # MechanicalDCFState: the rate path, flows, discount factors and terminal value the value reproduces from
                         "dcf_state": _tsm.get("state")}

    # Unit-integrity guard: a method implying >20x or <5% of the market price is a data
    # break (currency, split, share-count), not a view — drop it rather than let the cap
    # turn garbage into a confident-looking +80%.
    _absurd = [m for m, (p_, c_) in fair_values.items()
               if not (0.05 * current_price < p_ < 20.0 * current_price)]
    for m in _absurd:
        fair_values.pop(m)
    if _absurd:
        print(f"    ⚠ dropped implausible method(s) {_absurd} — unit/currency mismatch suspected")

    if not fair_values:
        return None

    # ── Lifecycle-weighted, confidence-weighted blend ──
    # Each method's weight = its own confidence × the Damodaran stage weight (young→revenue/DCF,
    # mature→earnings/FCF, decline→asset/yield), so the rating reflects the right tool for the stage.
    # The composite is PRICING (what the market's own yardsticks imply). The mechanical DCF is a
    # valuation cross-check and is kept OUT of the blend — otherwise intrinsic value leaks into
    # the number labelled "market composite" and gets double-counted against the story value.
    _pricing = {m: pc for m, pc in fair_values.items() if m != "DCF"} or dict(fair_values)
    _fw_excl = {}
    _pol_sel = (data or {}).get("policy")
    if _pol_sel is not None:
        # the applicability policy decides which pricing methods may run for this issuer type;
        # methods outside it are not down-weighted, they do not run
        for _mx in list(_pricing):
            if _mx not in _pol_sel.allowed_pricing_methods and len(_pricing) > 1:
                _fw_excl[_mx] = f"not applicable to a {_pol_sel.issuer_class} under the applicability policy — {_pol_sel.reason[:90]}"
                _pricing.pop(_mx, None)
    _wm = {m: c * _stage_w.get(m, 1.0) for m, (_p, c) in _pricing.items()}
    _wm_raw = dict(_wm)
    total_w = sum(_wm.values()) or 1.0
    fair_price = sum(p * _wm[m] for m, (p, _c) in _pricing.items()) / total_w
    _raw_composite = float(fair_price)
    _steps = []           # every adjustment after the raw weighted average, in order, for the audit
    _dropped = dict(_fw_excl)
    # Outlier trimming: a pricing method more than 2.5x or under 0.4x the median of ALL the pricing
    # methods is a basis break (a REIT-sector P/S median applied to a 4%-margin broker implied
    # $1,902 for JLL), not a view. Dropped, and shown as a step so the composite stays reproducible.
    # (The median of the OTHER methods let a majority of extreme methods expel the sane minority: INTU
    # 2026-09-24 dropped PEG and the analyst target at 1.2x while keeping P/E, P/FCF and P/S at 2-4.6x.)
    _outliers = {}
    if len(_pricing) >= 3:
        _pm_ = {m: p for m, (p, _c) in _pricing.items()}
        _med = float(np.median(list(_pm_.values())))
        for m, p in list(_pm_.items()):
            if p > 2.5 * _med or p < 0.4 * _med:
                _outliers[m] = f"{p / current_price:.1f}x price vs a median of {_med / current_price:.1f}x across the methods"
        if _outliers and len(_pricing) - len(_outliers) >= 2:
            for m in _outliers:
                _wm.pop(m, None); _pricing.pop(m, None)
            _dropped.update(_outliers)
            total_w = sum(_wm.values()) or 1.0
            fair_price = sum(p * _wm[m] for m, (p, _c) in _pricing.items()) / total_w
            _steps.append({"step": "outlier trimming", "set_to": float(fair_price),
                           "note": "dropped " + "; ".join(f"{m} ({why})" for m, why in _outliers.items())})
            for m in _outliers:
                _adj.append(_pst.applied("OUTLIER_TRIM", method=m, reason=_outliers[m]))

    market_cap = safe_float(overview.get("MarketCapitalization", 0)) or 0
    fv_ratio = fair_price / current_price if current_price > 0 else 1.0
    fv_flag = None
    if market_cap > 200_000_000_000:
        upper_cap, lower_cap = 1.60, 0.60
    elif market_cap > 50_000_000_000:
        upper_cap, lower_cap = 1.80, 0.55
    elif market_cap > 10_000_000_000:
        upper_cap, lower_cap = 2.00, 0.50
    else:
        upper_cap, lower_cap = 2.50, 0.40

    if daily is not None and len(daily) >= 252:
        ann_vol = float(
            daily["adj_close"].pct_change().rolling(252).std().iloc[-1] * np.sqrt(252)
        )
        if ann_vol < 0.25:
            upper_cap = min(upper_cap, 1.50)
            lower_cap = max(lower_cap, 0.65)
        elif ann_vol < 0.35:
            upper_cap = min(upper_cap, 1.80)

    industry = (overview.get("Industry") or "").lower()
    sector_raw = (overview.get("Sector") or "").upper()
    is_mreit = ("reit" in industry and "mortgage" in industry) or (
        sector_raw == "REAL ESTATE" and ("mortgage" in industry or "hybrid" in industry)
    )
    if is_mreit:
        upper_cap = min(upper_cap, 1.40)
        lower_cap = max(lower_cap, 0.70)

    # ── Compounder floor relief (2026) ───────────────────────────────────────
    # The lower_cap mean-reverts multiples toward sector medians, which systematically
    # slams high-P/E secular growers to the floor (LLY, ANET pinned at −40%). For genuine
    # compounders — high ROE, expanding/stable gross margin, revenue NOT at a cycle peak —
    # that compression fights a real structural-quality story. Raise the floor so FV can't
    # crater past −22%. Does NOT touch cyclicals or margin-compressors (the floor they
    # deserve). Symmetric in intent with the growth-premium on the upside.
    # Compounder = durable high ROE + non-compressing gross margin + still-growing EPS,
    # and NOT a commodity cyclical. We deliberately do NOT gate on revenue-vs-10yr-median
    # deviation: a genuine multi-year compounder ALWAYS sits far above its trailing median
    # (the median includes its small early years), so that test would wrongly exclude the
    # very names this relief targets. Cyclicality is already screened by `is_cyclical`, and
    # margin collapse by the gross-margin-trend gate.
    _roe = safe_float(overview.get("ReturnOnEquityTTM"))
    _gm_trend = pred_fund.get("gross_margin_trend_yoy")
    _eps_g_fv = pred_fund.get("eps_growth", 0) or 0
    _nd_ebitda_fv = pred_fund.get("net_debt_to_ebitda")
    _is_compounder = (
        not is_cyclical
        and _roe is not None and _roe > 0.18                  # durable high returns on equity
        and (_gm_trend is None or _gm_trend > -0.010)         # gross margin not compressing
        and _eps_g_fv > 0.03                                  # still growing earnings
        and (_nd_ebitda_fv is None or _nd_ebitda_fv < 4.0)    # not balance-sheet-stressed
    )
    if _is_compounder:
        lower_cap = max(lower_cap, 0.78)   # floor relief: worst-case FV ≈ −22% not −40%
        result_compounder_flag = "COMPOUNDER_FLOOR_RELIEF"
        _adj.append(_pst.applied("COMPOUNDER_FLOOR_RELIEF", lower=lower_cap))
    else:
        result_compounder_flag = None

    if fv_ratio > upper_cap:
        fv_flag = f"FV_CAPPED_HIGH (raw {fv_ratio:.2f}x, cap {upper_cap:.1f}x)"
        fair_price = current_price * upper_cap
    elif fv_ratio < lower_cap:
        fv_flag = f"FV_CAPPED_LOW (raw {fv_ratio:.2f}x, cap {lower_cap:.2f}x)"
        fair_price = current_price * lower_cap
    if fv_flag:
        _adj.append(_pst.applied("SANITY_CAP", upper=upper_cap, lower=lower_cap, raw_ratio=float(fv_ratio)))
        _steps.append({"step": "sanity cap", "set_to": float(fair_price),
                       "note": f"{fv_flag}: composite limited to {upper_cap:.2f}x / {lower_cap:.2f}x of the price for this market-cap and volatility tier"})

    raw_implied = (fair_price - current_price) / current_price if current_price > 0 else 0.0
    implied_return = float(np.clip(raw_implied, -0.90, 5.00))
    avg_conf       = float(np.mean([c for _, c in fair_values.values()]))

    # Reduce confidence when methods strongly disagree
    prices = [p for p, _ in fair_values.values()]
    if len(prices) > 1:
        cv = np.std(prices) / abs(np.mean(prices)) if np.mean(prices) != 0 else 0
        if cv > 0.30:
            avg_conf *= 0.80   # was 0.65 — relaxed so high-growth stocks with wide valuation spreads aren't over-penalised
            _adj.append(_pst.applied("METHOD_DISPERSION", factor=0.80, cv=float(cv)))

    # Build sector context for LLM visibility
    cur_pe_str   = f"{safe_float(overview.get('PERatio')):.1f}x" if safe_float(overview.get('PERatio')) else "N/A"
    cur_pfcf_str = f"{safe_float(overview.get('PriceToCashFlowRatio')):.1f}x" if safe_float(overview.get('PriceToCashFlowRatio')) else "N/A"
    cur_ps_str   = f"{current_ps:.1f}x" if current_ps else "N/A"
    sector_context = {
        "sector":      sector_name or "Unknown",
        "stock_pe":    cur_pe_str,
        "stock_pfcf":  cur_pfcf_str,
        "stock_ps":    cur_ps_str,
        "sector_pe":   f"{sector_meds['pe']:.0f}x",
        "sector_pfcf": (f"{sector_meds['pfcf']:.0f}x" if sector_meds.get("pfcf") else "N/A (no sourced peer P/FCF)"),
        "anchor_sources": {k: (v.get("source"), v.get("date"), v.get("universe")) for k, v in _anchor_src.items()},
        "sector_ps":   f"{sec_ps:.1f}x",
    }

    conf_out = min(avg_conf, 0.72)

    fv_quality_penalty = 1.0
    anchor_flag = pred_fund.get("eps_anchor_flag", "") or ""
    cyclical_adj = pred_fund.get("eps_cyclical_adjustment")
    fcf_inconsistency = pred_fund.get("eps_fcf_inconsistency") or ""

    if "HEAVY_ANCHOR" in anchor_flag:
        fv_quality_penalty *= 0.85
    if cyclical_adj and "HEAVY_DAMPEN" in cyclical_adj:
        fv_quality_penalty *= 0.75
    if fcf_inconsistency and "HIGH" in str(fcf_inconsistency):
        fv_quality_penalty *= 0.70

    # ── Quality fundamentals → confidence adjustment ─────────────────────────
    # gross-margin expansion = sustainable earnings power; R&D intensity = moat
    # proxy for tech/healthcare; net-debt/EBITDA = leverage stress beyond raw D/E.
    quality_signals = []
    gm_trend = pred_fund.get("gross_margin_trend_yoy")
    if gm_trend is not None:
        if gm_trend > 0.020:           # +200bps gross-margin expansion YoY — very strong
            fv_quality_penalty *= 1.10
            quality_signals.append(f"GM_EXPANDING ({gm_trend*100:+.0f}bps)")
        elif gm_trend > 0.005:         # +50bps — modestly positive
            fv_quality_penalty *= 1.04
        elif _value_trap_flag and gm_trend < -0.005:   # compression is already a priced deterioration signal
            quality_signals.append(f"GM_COMPRESSING ({gm_trend*100:+.0f}bps, priced under deterioration)")
        elif gm_trend < -0.020:        # -200bps — margin compression
            fv_quality_penalty *= 0.85
            quality_signals.append(f"GM_COMPRESSING ({gm_trend*100:+.0f}bps)")
        elif gm_trend < -0.005:
            fv_quality_penalty *= 0.95

    rd_int = pred_fund.get("rd_intensity")
    sector_tag = (overview.get("Sector") or "").upper()
    if rd_int is not None and sector_tag in ("TECHNOLOGY", "HEALTHCARE", "COMMUNICATION SERVICES"):
        # For innovation-driven sectors, sustained R&D is a moat indicator
        if rd_int > 0.15:              # >15% of revenue → very high reinvestment
            fv_quality_penalty *= 1.06
            quality_signals.append(f"HIGH_RD ({rd_int*100:.0f}%)")
        elif rd_int < 0.02 and sector_tag == "TECHNOLOGY":
            # Tech company with near-zero R&D — atypical, suggests low-moat business
            fv_quality_penalty *= 0.92

    nd_ebitda = pred_fund.get("net_debt_to_ebitda")
    if nd_ebitda is not None:
        if nd_ebitda > 5.0:            # severely overleveraged
            fv_quality_penalty *= 0.75
            quality_signals.append(f"OVER_LEVERED (ND/EBITDA={nd_ebitda:.1f}x)")
        elif nd_ebitda > 3.5:          # high leverage
            fv_quality_penalty *= 0.88
        elif nd_ebitda < 0.0:          # net cash position — balance-sheet fortress
            fv_quality_penalty *= 1.08
            quality_signals.append(f"NET_CASH ({-nd_ebitda:.1f}x EBITDA)")
        elif nd_ebitda < 1.0:          # clean balance sheet
            fv_quality_penalty *= 1.03

    # Cap the multiplier so quality alone can't push confidence past sensible bounds
    fv_quality_penalty = float(np.clip(fv_quality_penalty, 0.40, 1.25))

    if fv_quality_penalty != 1.0:
        conf_out = conf_out * fv_quality_penalty
        if fv_quality_penalty < 1.0:
            _adj.append(_pst.applied("FORECAST_QUALITY", factor=float(fv_quality_penalty)))

    if fv_flag:
        conf_out = min(conf_out * 0.45, 0.35)
        _adj.append(_pst.applied("SANITY_CAP_CONFIDENCE", factor=0.45))
    if _value_trap_flag:
        conf_out = conf_out * 0.80   # trap risk lowers confidence (its magnitude effect is the anchor shrink)
        _adj.append(_pst.applied("VALUE_TRAP_CONFIDENCE", factor=0.80, signals=list(_vt_signals)))
    result = {
        "fair_price":       fair_price,
        "expected_return":  implied_return,
        "confidence":       conf_out,
        "lifecycle_stage":  _stage,
        "lifecycle_basis":  _stage_basis,
        "dcf_meta":         _dcf_meta,
        "fv_eps_quality_penalty": float(fv_quality_penalty) if fv_quality_penalty != 1.0 else None,
        "fv_quality_signals": quality_signals if quality_signals else None,
        "fv_compounder_flag": result_compounder_flag,
        "fv_value_trap_flag": _value_trap_flag,
        "fv_sanity_flag":    fv_flag,
        "analyst_target_haircut": _at_note,
        "market_cap":        market_cap,
        "sector_context":   sector_context,
        "methods": {
            k: {"price": p, "conf": c,
                "implied_return": (p - current_price) / current_price,
                "in_composite": ((k != "DCF") or (len(fair_values) == 1)) and k not in _dropped,
                "dropped_reason": _dropped.get(k),
                "stage_weight": _stage_w.get(k, 1.0),
                "share": (_wm.get(k, 0.0) / total_w) if k in _wm else 0.0}
            for k, (p, c) in fair_values.items()
        },
        "composite_note": (f"market composite = Σ price × (confidence × stage weight) / Σ (confidence × stage weight) over the pricing methods, "
                           f"stage '{_stage}' weights {dict((k, round(_stage_w.get(k, 1.0), 2)) for k in _pricing)}; then outlier trimming, any value-trap haircut and the sanity caps — each shown as a step"),
        # Full audit trail: the user must be able to reproduce the published composite from this.
        "composite_audit": {
            "formula": "raw composite = Σ implied price × (confidence × lifecycle weight) / Σ (confidence × lifecycle weight) over the pricing methods (DCF excluded); then the steps in order",
            "stage": _stage,
            "methods": [{"method": k, "price": float(p), "implied_return": float((p - current_price) / current_price), "confidence": float(c),
                         "stage_weight": float(_stage_w.get(k, 1.0)),
                         "eff_weight": float(_wm_raw.get(k, 0.0)), "in_raw": k in _wm_raw,
                         "share_raw": float(_wm_raw.get(k, 0.0) / (sum(_wm_raw.values()) or 1.0)) if k in _wm_raw else 0.0,
                         "share_final": float(_wm.get(k, 0.0) / total_w) if k in _wm else 0.0,
                         "in_composite": k in _wm, "dropped_reason": _dropped.get(k)}
                        for k, (p, c) in fair_values.items()],
            "raw": _raw_composite, "raw_vs_price": float(_raw_composite / current_price - 1.0),
            "steps": _steps, "final": float(fair_price), "final_vs_price": float(fair_price / current_price - 1.0),
            "analyst_target": {"raw": analyst_target_ov, "used": fair_values.get("Analyst Target", (None,))[0], "note": _at_note},
            "cap_upper": float(upper_cap), "cap_lower": float(lower_cap),
            "method_lineage": _mult_lineage, "anchor_lineage": _anchor_src, "input_lineage": _input_lineage,
            "adjustments": _adj,
        },
        "pe_hist_median": hist_pe, "valuation_pe": valuation_pe,
    }

    analyst_target = safe_float(overview.get("AnalystTargetPrice"))
    if analyst_target and analyst_target > 0:
        result["analyst_target"] = analyst_target
        result["analyst_target_return"] = (analyst_target - current_price) / current_price
        if result["expected_return"] > 0.10 and result["analyst_target_return"] > 0.10:
            result["confidence"] = min(result["confidence"] * 1.10, 0.85)

    n_methods = len(fair_values)
    if n_methods == 1:
        result["confidence"] = result["confidence"] * 0.50
        result["fv_method_warning"] = f"SINGLE_METHOD ({list(fair_values.keys())[0]})"
    elif n_methods == 2:
        result["confidence"] = result["confidence"] * 0.75
        result["fv_method_warning"] = f"LIMITED_METHODS ({n_methods})"

    pred_eps = pred_fund.get("eps_annual", 0) or 0
    ttm_eps = pred_fund.get("eps_ttm_current", 0) or 0
    if ttm_eps < 0 and pred_eps < 0:
        result["confidence"] = result["confidence"] * 0.60
        result["fv_pre_profit_flag"] = True
    elif ttm_eps < 0 and pred_eps > 0:
        result["confidence"] = result["confidence"] * 0.80
        result["fv_turnaround_flag"] = True

    return result


def analyze_fundamental_ml_yearly(data, current_price, state=None):
    """
    Orchestrates Stage 1 (predict fundamentals) + Stage 2 (derive fair value).
    Returns a blend-ready dict compatible with blend_predictions_yearly.

    With an AnalysisState, the raw model outputs pass through the single reconciliation stage
    (state_builder.reconcile_forecast_from_state) before pricing, and pricing consumes only the
    ReconciledForecastState and canonical metrics (pricing_inputs.canonical_view).
    """
    try:
        import canonical as _cnc
        for _n in _cnc.normalize_consensus(data):
            print(f"  {YELLOW}[consensus]{RESET} {_n}")
    except Exception as _ce:  # noqa: BLE001
        print(f"  {YELLOW}[consensus] normalization skipped: {str(_ce)[:70]}{RESET}")
    pred_fund = predict_fundamentals_ml(data)
    if not pred_fund:
        return None, None

    try:
        import canonical as _cnf
        _fnotes = _cnf.normalize_forecasts(pred_fund, data)
        for _n in _fnotes:
            print(f"  {YELLOW}[units]{RESET} {_n}")
    except Exception as _ue:  # noqa: BLE001
        print(f"  {YELLOW}[units] forecast normalization skipped: {str(_ue)[:70]}{RESET}")
    if state is not None:
        import state_builder as _sbr
        pred_fund["_reconciled"] = _sbr.reconcile_forecast_from_state(state, pred_fund)
        for _a in (pred_fund["_reconciled"].get("reconciliation") or {}).get("adjustments") or []:
            print(f"  {YELLOW}[reconcile]{RESET} {str(_a)[:110]}")
    fair_val = derive_fair_value(current_price, pred_fund, data, state=state)
    if not fair_val:
        return pred_fund, None

    # Attach predicted fundamentals to fair_val for downstream use
    fair_val["pred_fund"] = pred_fund

    return pred_fund, fair_val


# ─── LLM Debate (Yearly-specific prompts) ───────────────────────────────────


_LLM_SYSTEM_PROMPT_YEARLY = """You are an expert fundamental equity analyst. You are given the outputs of quantitative models predicting a stock's 1-YEAR return. Your primary inputs are:
1. FUNDAMENTAL SCORECARD — current financial health
2. PREDICTED FUNDAMENTALS — ML-forecasted EPS, revenue, margins, FCF for next 12 months
3. FUNDAMENTAL FAIR VALUE — multi-method valuation (P/E, PEG, P/FCF, P/S) using predicted fundamentals
4. MACRO environment (yield curve, rates, CPI)
5. SENTIMENT & INSIDER activity (qualitative context)
6. DIRECT PRICE ML (ensemble — blended at fixed weight with fair value and macro)

Key principles for 1-YEAR predictions:
- VALUATION drives returns: a stock at fair value returns ~market rate; below fair value outperforms
- EPS GROWTH trajectory is the single most important fundamental input
- MARGIN EXPANSION is strongly bullish; margin contraction is a red flag
- INSIDER BUYING has the highest alpha signal in the 6-12 month window
- YIELD CURVE: inversion predicts recessions 6-18 months ahead — be cautious
- ML direct-price predictions have fewer independent samples at 1yr — be skeptical of extremes
- Most large-cap stocks return 5-15% annually. Predictions >30% need strong justification.
- When fundamental fair value methods disagree strongly, widen your confidence interval

*** GROWTH-STOCK CALIBRATION (critical for high-growth names) ***
- A stock can be OVERVALUED on current multiples but still APPRECIATE if earnings growth is fast enough.
  A 250x P/E stock growing EPS 70% YoY will be ~147x P/E in 12 months even at 0% price change. Use FORWARD multiples, not trailing.
- PEG < 1.0 means the market may be UNDERPRICING growth. Do not automatically SELL a stock just because trailing P/E is high if PEG is below 1.0.
- Stocks with >40% revenue growth decline >25% in only ~15-20% of years. STRONG SELL (>15% decline) requires a specific catalyst or structural break, not just "multiples are high."
- The BIGGEST analytical error on growth stocks is anchoring on trailing multiples while ignoring how fast growth compresses those multiples forward.
- Consider THREE scenarios: (a) multiple compression to sector norms, (b) partial compression with earnings growth, (c) multiple sustained by growth.
  Your prediction should reflect probability weighting, not only the worst case.

Recommendation guidelines:
- STRONG BUY: >15% upside with fair value support + EPS growth + insider buying
- BUY: >5% upside with reasonable fundamental support
- HOLD: -5% to +5%, or high uncertainty / conflicting signals
- SELL: >5% downside with fundamental deterioration OR extreme overvaluation without sufficient growth to compress multiples within 2-3 years
- STRONG SELL: >15% downside with SPECIFIC deterioration — earnings miss, guidance cut, margin collapse, or structural demand decline.
  High multiples alone do NOT justify STRONG SELL if growth is sustained.

You MUST respond with valid JSON only. No markdown, no explanation outside JSON."""


_SMART_MONEY_CACHE = {"loaded": False, "df": None}






_PANEL_ML_CACHE = {"loaded": False, "df": None}


def _crucible_panel_ml_rank(ticker):
    """Cross-sectional panel-ML rank (12-month horizon) from the shipped snapshot. Per Crucible's
    own prediction backtest this cross-sectional factor model is the highest-signal quant view
    (and far better than the single-stock ML). Returns a debate-ready text block or '' — guarded."""
    import os
    try:
        if not _PANEL_ML_CACHE["loaded"]:
            _PANEL_ML_CACHE["loaded"] = True
            p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "panel_ml_snapshot.parquet")
            if os.path.exists(p):
                import pandas as pd
                _PANEL_ML_CACHE["df"] = pd.read_parquet(p).set_index("ticker")
        df = _PANEL_ML_CACHE["df"]
        if df is None or ticker not in df.index:
            return ""
        r = df.loc[ticker]
        pred = float(r.get("pred_excess_12m", 0.0)) * 100.0
        pctl = float(r.get("xs_rank_pct", 50.0))
        dec = int(r.get("decile", 5))
        asof = str(r.get("as_of", ""))
        n = int(r.get("n_universe", 0))
        tilt = ("top decile" if dec >= 9 else "bottom decile" if dec <= 2 else
                "upper half" if pctl >= 55 else "lower half" if pctl <= 45 else "middle of the pack")
        import policy as _polp
        _rel = _panel_state(ticker)[0]
        if not _rel.get("persona_visibility"):
            return ("CROSS-SECTIONAL QUANT RANK — reliability %s: NOT USABLE AS EVIDENCE. %s. The rank and its "
                    "expected relative return are withheld from this brief and kept in diagnostics."
                    % (_rel["state"], _rel["basis"]))
        return ("CROSS-SECTIONAL QUANT RANK (12-month panel-ML across the top-%d universe) — reliability %s (%s):\n"
                "  %s sits in decile %d/10 (%.0fth percentile, %s); the model expects %+.1f%% return RELATIVE to the "
                "universe over 12 months (as-of %s). It is a value+quality+momentum ranking, not a price target.\n"
                "  You may describe it only as %s. FORBIDDEN wording: %s."
                % (n, _rel["state"], _rel["basis"], ticker, dec, pctl, tilt, pred, asof,
                   _rel["allowed_claim_strength"], ", ".join(_rel["forbidden_phrases"])))
    except Exception:
        return ""


_SMART_MONEY_CACHE = {"loaded": False, "df": None}


def _ownership_13f(ticker):
    """Structured 13F ownership for one ticker from the shipped snapshot (built weekly from Sharadar
    SF3 by the refresh chain). Returns None when the snapshot or the ticker is absent."""
    try:
        if not _SMART_MONEY_CACHE["loaded"]:
            _SMART_MONEY_CACHE["loaded"] = True
            p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "smart_money_snapshot.parquet")
            if os.path.exists(p):
                import pandas as pd
                _SMART_MONEY_CACHE["df"] = pd.read_parquet(p).set_index("ticker")
        df = _SMART_MONEY_CACHE["df"]
        if df is None or ticker not in df.index:
            return None
        r = df.loc[ticker]

        def num(k):
            v = safe_float(r.get(k))
            return None if (v is None or v != v) else v

        def txt(k):
            v = r.get(k)
            return None if v is None or (isinstance(v, float) and v != v) else str(v)
        as_of = r.get("as_of_13f")
        return {"as_of": (str(as_of)[:10] if as_of is not None and str(as_of) not in ("NaT", "nan") else None),
                "n_holders": num("n_holders"), "n_holders_prev": num("n_holders_prev"),
                "n_gurus": num("n_gurus"), "guru_new_buys": num("guru_new_buys"), "guru_adds": num("guru_adds"),
                "guru_trims": num("guru_trims"), "guru_exits": num("guru_exits"),
                "guru_names": txt("guru_names"), "guru_buy_names": txt("guru_buy_names"),
                "guru_exit_names": txt("guru_exit_names"),
                "source": "SEC Form 13F holdings (Sharadar SF3), curated long-term investor list"}
    except Exception:  # noqa: BLE001
        return None


def canonical_risk_free(all_data):
    """The 10-year Treasury yield the data stage validated (D9) — the one risk-free source."""
    t = (all_data or {}).get("treasury_10y") or []
    v = safe_float(t[0].get("value")) if (isinstance(t, list) and t and isinstance(t[0], dict)) else None
    return (v / 100.0) if v is not None else None


def _damodaran_inputs(ticker, all_data, sic=None):
    """The company's inputs from Damodaran's datasets, assembled once per run and read by every model:
    his industry (company list), the industry's cash-corrected unlevered beta re-levered at the
    company's market D/E (bottom-up beta), his implied ERP for the month plus the country risk premium
    of the country of operations, and the industry's sales-to-capital and operating margins."""
    import damodaran_data as dd
    if not dd.available():
        return {"available": False, "reason": "damodaran_data.json.gz missing"}
    ov = all_data.get("overview") or {}
    iss = all_data.get("issuer") or {}
    ind, ctry, how = dd.company_industry(ticker, ov.get("Name") or iss.get("name"), sic)
    country = ctry or (iss.get("country") if iss.get("is_fpi") else "United States") or "United States"
    region = "US" if country == "United States" else "Global"
    stats = dd.industry_stats(ind, region) or dd.industry_stats(ind, "US")
    bq = (((all_data.get("balance") or {}).get("quarterlyReports") or [{}])[0]) or {}
    debt = (safe_float(bq.get("shortLongTermDebtTotal")) or
            ((safe_float(bq.get("longTermDebt")) or safe_float(bq.get("longTermDebtNoncurrent")) or 0.0) +
             (safe_float(bq.get("shortTermDebt")) or safe_float(bq.get("currentDebt")) or 0.0)))
    mcap = safe_float(ov.get("MarketCapitalization"))
    beta = dd.bottom_up_beta(stats, debt, mcap, tax=0.25)
    unlevered = (stats or {}).get("unlevered_beta_cash")
    s2c = (stats or {}).get("sales_to_capital")
    # his business-mix approach: a company with several business lines is as risky as the value-weighted
    # mix of its businesses, not as its single classified industry (SpaceX: launch, broadband, AI)
    # industries are assigned only after the segments reconcile to the company (segments.canonicalize), and
    # the business mix reads nothing else: a segment list that fails an invariant leaves the single-industry beta
    import segments as _sgm
    # the valuation perimeter's segments: after a separation, the continuing company's (segments.for_perimeter)
    canon = (all_data.get("segments_forward") or all_data.get("segments_canonical")
             or {"status": "UNAVAILABLE", "reason": "segments not canonicalized"})
    # each segment's industry comes through a SegmentIndustryMapping: the keyword map, then the parent-sector
    # compatibility gate and its fallback (a non-financial company's segment is never priced as a financial business
    # without a financial business named in the segment's own text)
    _fin = bool(iss.get("is_financial"))
    _sgm.allocate(canon, ind, lambda n, pl, fb: dd.segment_industry(n, pl, fb),
                  mapper=lambda sid, text: dd.map_segment(sid, text, (), ind, _fin, use_product_lines=False))
    bm = (dd.business_mix(canon.get("segments"), region, ind, debt, mcap, parent_is_financial=_fin)
          if canon.get("business_mix_allowed") else None)
    # BUSINESS_MIX_BETA_VALID: every industry the mix used must have passed compatibility; otherwise the single-industry
    # beta applies (never a precision blend over an incompatible mapping)
    _bm_bad = dd.business_mix_problems((bm or {}).get("segments"), ind, _fin) if bm else []
    if _bm_bad:
        bm = None
    bm_status = ("used" if bm else (("rejected: " + "; ".join(_bm_bad)[:200]) if _bm_bad else
                                    "single industry" if canon.get("business_mix_allowed")
                                    else f"disabled: segments {canon.get('status')} ({str(canon.get('reason'))[:160]})"))
    beta_basis = f"industry {ind}" if ind else "no industry match"
    if bm:
        beta, unlevered, s2c = bm["beta"], bm["unlevered_beta"], bm["sales_to_capital"] or s2c
        beta_basis = (f"business mix of {len(bm['segments'])} filed segments, weighted by estimated value (revenue × industry EV/Sales)"
                      if bm.get("weighting") == "value" else f"business mix of {len(bm['segments'])} filed segments (revenue-weighted)")
    erp = dd.company_erp(country)
    # stable period: a mature company's cost of capital at today's riskfree rate and this company's ERP
    stable = dd.mature_cost_of_capital(canonical_risk_free(all_data), (erp or {}).get("erp"), region)
    return {"available": True, "industry": ind, "industry_match": how, "region": region, "country": country,
            "unlevered_beta": unlevered, "debt_to_equity": (debt / mcap) if (debt and mcap) else 0.0,
            "beta": beta, "beta_basis": beta_basis, "business_mix": (bm or {}).get("segments"),
            "business_mix_weighting": (bm or {}).get("weights"), "business_mix_status": bm_status,
            "segments_status": canon.get("status"),
            "stable_cost_of_capital": (stable or {}).get("value"), "stable_cost_of_capital_basis": (stable or {}).get("basis"),
            "stable_market_beta": (stable or {}).get("beta"),
            "industry_sales_to_capital": s2c,
            "industry_operating_margin": (stats or {}).get("operating_margin"),
            "industry_operating_margin_adj": (stats or {}).get("operating_margin_adj"),
            "industry_as_of": (stats or {}).get("as_of"), "n_firms": (stats or {}).get("n_firms"),
            "industry_multiples": dd.industry_multiples(ind, region) if ind else {},
            "erp": (erp or {}).get("erp"), "implied_erp": (erp or {}).get("implied_erp"), "crp": (erp or {}).get("crp"),
            "rating": (erp or {}).get("rating"), "erp_month": (erp or {}).get("erp_month"), "crp_as_of": (erp or {}).get("crp_as_of"),
            "erp_basis": (erp or {}).get("basis"), "source": "Aswath Damodaran, pages.stern.nyu.edu/~adamodar (datasets)"}


def _gaap_distortion_check(all_data, mechanical_stage):
    """Deterministic GAAP-distortion test on the reconciled figures: reported EPS (reconciled TTM net
    income ÷ shares) versus the provider's adjusted EPS. A gap above 40%, or a reported loss beside an
    adjusted profit, means statement-earnings multiples misstate the business. The lifecycle stage is
    the mechanical classifier's; no model can override it."""
    rh = all_data.get("reconciled_historicals") or {}
    sh = safe_float((all_data.get("overview") or {}).get("SharesOutstanding"))
    ni = rh.get("net_income_ttm")
    q = ((all_data.get("earnings") or {}).get("quarterlyEarnings") or [])[:4]
    adj = [safe_float(x.get("reportedEPS")) for x in q]
    adj_ttm = sum(adj) if len(adj) == 4 and all(v is not None for v in adj) else None
    rep = (ni / sh) if (ni is not None and sh) else None
    flag, why = False, "reported and adjusted EPS agree within 40% (or one is unavailable)"
    if rep is not None and adj_ttm and adj_ttm > 0:
        gap = rep / adj_ttm - 1.0
        if rep <= 0 or abs(gap) > 0.40:
            flag = True
            why = (f"reported EPS {rep:.2f} vs adjusted EPS {adj_ttm:.2f} ({gap:+.0%}) on the reconciled trailing figures")
    return {"mechanical": mechanical_stage, "gaap_distortion": flag, "distortion_reason": why, "overridden": False,
            "basis": "deterministic: reconciled reported EPS vs adjusted EPS"}


def _panel_state(ticker):
    """(reliability, row) for the cross-sectional panel rank. Reliability comes only from the
    snapshot's validation sidecar (panel_ml_snapshot.meta.json); without it the model is UNAVAILABLE."""
    import policy as _polp
    meta = None
    try:
        mp = os.path.join(os.path.dirname(os.path.abspath(__file__)), "panel_ml_snapshot.meta.json")
        if os.path.exists(mp):
            meta = json.loads(open(mp).read()).get("validation")
    except Exception:  # noqa: BLE001
        meta = None
    rel = _polp.panel_reliability(meta)
    row = None
    try:
        _crucible_panel_ml_rank(ticker)
        df = _PANEL_ML_CACHE.get("df")
        if df is not None and ticker in df.index:
            r = df.loc[ticker]
            row = {"decile": int(r.get("decile", 5)), "xs_rank_pct": float(r.get("xs_rank_pct", 50.0)),
                   "pred_excess_12m": float(r.get("pred_excess_12m", 0.0)), "as_of": str(r.get("as_of", "")),
                   "n_universe": int(r.get("n_universe", 0))}
    except Exception:  # noqa: BLE001
        row = None
    return rel, row






def _call_claude_yearly(user_msg, system=None, verbose=False):
    """Claude caller for yearly debate — uses 800 max_tokens for longer fundamental
    signal prompts (shared _call_claude caps at 500, which can truncate JSON)."""
    if not ANTHROPIC_API_KEY:
        return None
    try:
        import anthropic
        # Explicit network timeout: SDK default is ~600s, which would hang the
        # debate for 10 minutes on an unreachable endpoint (mirrors _LLM_TIMEOUT
        # used by the shared callers in predict_monthly).
        client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY, timeout=120.0)
        response = claude_create(client, 
            model="claude-sonnet-5",
            max_tokens=2400,   # structured-argument JSON (nested points) needs more than the old 800
            system=system or _LLM_SYSTEM_PROMPT_YEARLY,
            messages=[{"role": "user", "content": user_msg}],
            temperature=1.0,
        )
        raw = response.content[0].text.strip()
        ti  = response.usage.input_tokens
        to  = response.usage.output_tokens
        if verbose:
            print(f"      {DIM}Claude Opus tokens: {ti} in / {to} out{RESET}")
        return _parse_llm_response(raw, ti, to, "claude-sonnet-5")
    except Exception as e:
        if verbose:
            print(f"      {DIM}Claude error: {str(e)[:80]}{RESET}")
        return None


# Yearly PM synthesis template — the yearly debate has NO blind Round-1 PM pass
# (the monthly path's {pm_r1_*} baseline). It anchors on the Neutral analyst instead.
# The shared monthly template requires {pm_r1_*}; formatting it with the yearly
# kwargs raised KeyError, silently collapsing the PM to an equal-weight fallback.
_PM_SYNTHESIS_USER_TEMPLATE_YEARLY = _STYLE_RULES + _EVIDENCE_RULES.replace("{", "{{").replace("}", "}}") + """STOCK: {ticker} | CURRENT: ${current_price:.2f} | HORIZON: {months}-month

CROSS-SECTIONAL QUANT RANK (12-month panel-ML). Its standing is set by the reliability record
below, not by this prompt; weight it no more strongly than that record allows, and if it is marked
NOT USABLE AS EVIDENCE you may not cite it for the recommendation or let it move the odds:
{panel_ml}
VALUE vs PRICE — canonical anchors (use these numbers and names; the advocates may have mislabelled them):
{anchors}
The STORY VALUE is the intrinsic value. The mechanical DCF is only a formula cross-check of it — never quote the
mechanical-DCF gap as "the intrinsic gap" or "the premium to fair value". The market composite is PRICING (what the
market's own yardsticks imply), not value. Treat the story-value-vs-price gap as a first-class input — your synthesis
MUST say explicitly whether you expect that gap to close toward value, toward price, or persist, and why.

NEUTRAL ANALYST (objective baseline — heard before the advocates):
  Return: {neutral_return:+.2f}% | Recommendation: {neutral_rec} | Confidence: {neutral_conf:.0f}%
  Reasoning: {neutral_reasoning}

RISK MANAGER (Bear) says:
  Return: {bear_return:+.2f}% → ${bear_price:.2f} | Recommendation: {bear_rec} | Confidence: {bear_conf:.0f}%
  Thesis: {bear_reasoning}
  Key Risks: {bear_risks}

GROWTH ANALYST (Bull) says:
  Return: {bull_return:+.2f}% → ${bull_price:.2f} | Recommendation: {bull_rec} | Confidence: {bull_conf:.0f}%
  Thesis: {bull_reasoning}
  Key Catalysts: {bull_catalysts}

Assign probability weights to each scenario. Judge the FULL mosaic, not just valuation/financials:
the business & growth outlook, competitive position & moat, key business/operational risks,
management & capital allocation, institutional & insider positioning (13F/Form-4) and news catalysts.
Constraints: p_bear ∈ [0.10, 0.65],  p_bull ∈ [0.10, 0.65],  p_neutral = 1 - p_bear - p_bull ≥ 0.05

HOW TO WRITE IT — you are writing the DECISION, not minutes of the meeting:
- "reasoning" is the decision thesis in 2-3 sentences (max 70 words): the verdict, the decisive
  evidence behind it, and where the 12-month expected return sits relative to the gap between the
  story value and the market price (closing toward value, toward price, or persisting) and why.
  Do NOT recap what the bear or the bull said — state what you concluded.
- "key_points" are WHAT DECIDED THE ODDS: at most 4 items. "topic" = a 2-5 word label of the
  deciding factor (your own words, e.g. "Insider selling", "AI capex payback", "Value gap");
  "point" = the conclusion, the figure that settles it (as its [[TOKEN]]), and how it moved the odds (one or two
  sentences). Attribute a point to an analyst ONLY when the fact that it went unanswered is itself
  the reason it decided.
- "triggers" = 1-3 STRUCTURED conditions that would move the verdict. Each is an object, not prose:
  metric (the measurable thing, e.g. "operating margin", "net charge-off ratio", "data-centre bookings growth"),
  direction ("up" or "down"), operator (">=" or "<="), threshold (one number with its unit: "6%", "$120", "25x"),
  current (today's level if you know it), scenario ("BULL" or "BEAR" — where the view moves if it happens),
  rationale (<= 12 words). Deterministic code checks that the economics of the metric match the destination,
  so state the direction honestly; do not phrase a deterioration as a bull trigger.
- "tipping_point" = one sentence restating the most important trigger in plain words.

Respond with ONLY this JSON:
{{
  "p_bear": <float>,
  "p_bull": <float>,
  "p_neutral": <float>,
  "conviction": <float: 0.0-1.0, confidence in these weights>,
  "reasoning": "<decision thesis, 2-3 sentences>",
  "key_points": [
    {{"topic": "<deciding factor>", "point": "<conclusion + the deciding figure as its [[TOKEN]] + effect on the odds>"}}
  ],
  "tipping_point": "<what would flip the verdict>",
  "triggers": [
    {{"metric": "<measurable>", "direction": "up|down", "operator": ">=|<=|beyond",
      "source_claim": "<the TOKEN name the threshold is measured against, e.g. GUIDANCE_9, or null>",
      "threshold": "<number in the SAME unit as that token's value: 35 for a value shown in bps, 6.0 for a value shown in %>",
      "current": "<today>", "scenario": "BULL|BEAR", "rationale": "<<=12 words>"}}
  ],
  "risk_flags": ["<combined key risks from both cases>"],
  "evidence_used": ["<ids from: {evidence_ids}>"],
  "quant_tilt": {{"bear": 0.0, "neutral": 0.0, "bull": 0.0}},
  "override_ml": <boolean>
}}
NOT ELIGIBLE as evidence for the odds (you may mention them as context, never list them): {evidence_ineligible}.
"evidence_used" lists ONLY the evidence your odds actually rest on. "quant_tilt" is how much (in
probability) a quantitative model moved each scenario; leave it at zero unless the brief marks that
model usable as evidence."""














def analyze_llm_debate_yearly(
    ticker,
    current_price,
    tech,
    fund,
    sent,
    macro,
    ml,
    pred_fund=None,
    fair_val=None,
    insider=None,
    data=None,
    verbose=False,
    state_mode=None,
):
    """4-role persona debate for yearly predictions.

    Round 1 — Neutral Analyst (Gemini 3 Flash): objective data-driven view.
    Round 2 — Bear (Kimi) + Bull (GPT): independent advocate positions, both see neutral view.
    Round 3 — Bear vs Bull debate: each attacks the other, defends own thesis.
    Round 4 — Portfolio Manager (Claude): assigns P(bear)/P(bull)/P(neutral).
               Final return computed mechanically from probability weights.

    ``state_mode`` is required: the brief, the PM anchors and the panel line come from the validated
    AnalysisState, every prompt carries the number-token rules, and nothing is cached or streamed.
    Prose is ingested once, downstream, from the frozen state (engine/prose.py)."""
    _sp_audit = None; _ev_audit = None   # §15/§16 audits; None when the panel did not run
    # ── state mode: the brief is built from the validated AnalysisState, every prompt carries the
    #    number-token rules, nothing is cached, no raw persona prose is streamed, and no correction
    #    pass runs here (prose is ingested once, downstream, from the frozen state) ──
    if not state_mode:
        raise ValueError("analyze_llm_debate_yearly requires state_mode (the brief built from AnalysisState)")
    _sm = state_mode
    _hdr = _sm.get("token_header") or ""
    # ── structured arguments (engine/arguments.py): when the pipeline supplies an argument context, authors cite
    #    evidence ids and write figure-free judgement; each round is validated and ONLY validated objects are forwarded
    _AC = _sm.get("args_ctx")
    _arg_drops, _args = [], {}
    if _AC is not None:
        import arguments as _A
        import panel_prompts as _PP
        from state_builder import rating_for as _rating_for
        _hdr = ""                                       # no token header: the evidence menu replaces it
        _menu = _sm.get("evidence_menu") or ""
        _brief2 = _sm.get("brief_v2") or _sm["brief"]

        def _own_r(ret):
            return _A._own(ret, _rating_for(round(float(ret), 3)) if ret is not None else None)

        def _settle(fn, *a, **k):
            """A validation that raises never kills the debate: the seat's objects are dropped with the reason."""
            try:
                return fn(*a, **k)
            except Exception as _e:  # noqa: BLE001
                _arg_drops.append({"source": "debate", "raw": None,
                                   "reason": f"INTERNAL_ERROR: {type(_e).__name__}: {str(_e)[:160]}"})
                return None, []

        def _numbers(name, r):
            return (f"{name}: return {r['expected_return'] * 100:+.1f}% ({_rating_for(round(float(r['expected_return']), 3))}), "
                    f"confidence {float(r.get('confidence') or 0.5) * 100:.0f}%")

        def _repair(call, prompt, system, raw, argset, drops, validate):
            """One bounded repair turn (engine/arguments.py): the author sees its removed objects with the typed reasons
            and returns the full JSON again. Only the repaired reply's structured part is used (every number stays the
            first reply's); it is validated exactly like the first, and kept only when it loses no object. Repairs stop
            once the run has used its repair budget (the server kills a run at its hard cap)."""
            fix = _A.repairable(drops)
            if not fix or call is None or not raw or argset is None or not _A.is_structured(raw):
                return argset, drops                  # a legacy-format reply is owned by the legacy path: never repaired
            if _time_mod.monotonic() - _PROCESS_T0 > float(os.getenv("CRUCIBLE_REPAIR_BUDGET_S", "330")):
                return argset, drops + [{"source": "repair", "raw": None, "reason": "REPAIR_SKIPPED: run time budget used"}]
            try:
                kw = {"system": system} if system is not None else {}
                rep = call(_A.repair_prompt(prompt, raw.get("structured"), fix), verbose=verbose, **kw)
            except Exception as _e:  # noqa: BLE001
                return argset, drops + [{"source": "repair", "raw": None,
                                         "reason": f"REPAIR_UNAVAILABLE: {type(_e).__name__}: {str(_e)[:100]}"}]
            st2 = (rep or {}).get("structured")
            if not isinstance(st2, dict) or not st2:
                return argset, drops + [{"source": "repair", "raw": None, "reason": "REPAIR_UNAVAILABLE: no structured reply"}]
            merged = dict(raw)
            merged["structured"] = st2
            a2, d2 = _settle(validate, merged)
            if a2 is not None and _A.repair_keeps(argset, a2):
                first = [dict(d, source=f"attempt1:{d.get('source')}", reason=f"REPAIR_REQUESTED: {d.get('reason')}")
                         for d in fix]
                return a2, first + list(d2)
            return argset, drops + [{"source": "repair", "raw": None,
                                     "reason": "REPAIR_NOT_KEPT: the repaired reply had fewer valid objects"}]
    # guidance instructions exist only when verified company guidance does (guidance.py)
    _gav = bool(_sm.get("guidance_available"))
    _r13 = bool(_sm.get("ownership_return_signal"))
    _ep = lambda p: _evidence_prompt(_guidance_prompt(p, _gav), _r13)
    _BEAR_SYSTEM_PROMPT = _ep(globals()["_BEAR_SYSTEM_PROMPT"])
    _BULL_SYSTEM_PROMPT = _ep(globals()["_BULL_SYSTEM_PROMPT"])
    _NEUTRAL_SYSTEM_PROMPT = _ep(globals()["_NEUTRAL_SYSTEM_PROMPT"])
    _BEAR_USER_TEMPLATE = _ep(globals()["_BEAR_USER_TEMPLATE"])
    _BULL_USER_TEMPLATE = _ep(globals()["_BULL_USER_TEMPLATE"])
    _NEUTRAL_USER_TEMPLATE = _ep(globals()["_NEUTRAL_USER_TEMPLATE"])

    def _with_tokens(fn):
        return lambda prompt, *a, **k: fn(_hdr + "\n\n" + prompt, *a, **k)
    _call_kimi = _with_tokens(globals()["_call_kimi"])
    _call_openai = _with_tokens(globals()["_call_openai"])
    _call_claude = _with_tokens(globals()["_call_claude"])
    _call_claude_yearly = _with_tokens(globals()["_call_claude_yearly"])
    _call_gemini_3_pro = _with_tokens(globals()["_call_gemini_3_pro"])
    _call_claude_synthesis = _with_tokens(globals()["_call_claude_synthesis"])
    signals_text = _sm["brief"]
    if _AC is not None:
        _NEUT_U, _NEUT_S = _PP.seat_r1("neutral", ticker, _brief2, _menu), _PP.system("neutral")
    else:
        _NEUT_U = _NEUTRAL_USER_TEMPLATE.format(ticker=ticker, months=12, current_price=current_price,
                                                signals_text=signals_text)
        _NEUT_S = _NEUTRAL_SYSTEM_PROMPT

    has_gemini = bool(GEMINI_API_KEY)
    has_kimi   = bool(KIMI_API_KEY)
    has_openai = bool(OPENAI_API_KEY)
    has_claude = bool(ANTHROPIC_API_KEY)
    total_tokens_in = total_tokens_out = 0

    available = []
    if has_gemini: available.append("gemini")
    if has_kimi:   available.append("kimi")
    if has_openai: available.append("gpt")
    if has_claude: available.append("claude")

    if len(available) < 2:
        if not available:
            return None
        print(f"    {YELLOW}Only 1 LLM available — single judge mode{RESET}")
        call_fn = (_call_gemini_3_pro if has_gemini else
                   _call_kimi         if has_kimi   else
                   _call_openai       if has_openai else
                   _call_claude_yearly)
        try:
            r = call_fn(_NEUT_U, system=_NEUT_S, verbose=verbose)
            if r:
                r["round1"] = {}; r["round2"] = {}
                r["vote"] = {"direction": r["direction"], "count": 1, "total": 1, "unanimous": True}
                r["rec_vote"] = {"recommendation": r.get("recommendation", "HOLD"),
                                 "count": 1, "total": 1, "unanimous": True}
                return r
        except Exception as e:
            print(f"    {RED}LLM error: {e}{RESET}")
        return None

    # ── Round 1: Neutral Analyst (Gemini 3 Flash) ────────────────────────────
    neutral_result = None
    _neut_call = _call_gemini_3_pro
    print(f"\n    {DIM}Round 1 — Neutral Analyst (objective view){RESET}")
    if has_gemini:
        try:
            neutral_result = _call_gemini_3_pro(_NEUT_U, system=_NEUT_S, verbose=verbose)
            if neutral_result:
                total_tokens_in  += neutral_result.get("tokens_in", 0)
                total_tokens_out += neutral_result.get("tokens_out", 0)
                rc = GREEN if neutral_result["expected_return"] > 0 else RED
                print(f"      {'Gemini (Neutral)':<20} {rc}{neutral_result['expected_return']*100:+.1f}%{RESET} "
                      f"{neutral_result.get('recommendation','HOLD'):<11} "
                      f"conf:{neutral_result['confidence']*100:.0f}%")
            else:
                print(f"      {'Gemini (Neutral)':<20} {YELLOW}parse failed — retrying the seat via fallback model{RESET}")
                # The Neutral seat must not stay empty (audit: AVGO ran a 3-voice panel that
                # still reported complete). Retry once through whichever other model is up.
                _nb = (_call_openai if has_openai else _call_claude_yearly if has_claude else
                       _call_kimi if has_kimi else None)
                if _nb:
                    try:
                        _neut_call = _nb
                        neutral_result = _nb(_NEUT_U, system=_NEUT_S, verbose=verbose)
                        if neutral_result:
                            rc = GREEN if neutral_result["expected_return"] > 0 else RED
                            print(f"      {'Neutral (fallback)':<20} {rc}{neutral_result['expected_return']*100:+.1f}%{RESET} "
                                  f"{neutral_result.get('recommendation','HOLD'):<11}")
                    except Exception as _ne:  # noqa: BLE001
                        print(f"      {'Neutral (fallback)':<20} {RED}{str(_ne)[:60]}{RESET}")
        except Exception as e:
            print(f"      {'Gemini (Neutral)':<20} {RED}{type(e).__name__}: {str(e)[:80]}{RESET}")

    if neutral_result and _AC is not None:
        _vn = lambda rep: _A.validate_seat(_AC, "neutral", rep, own=_own_r(neutral_result["expected_return"]))
        _na, _nd = _settle(_vn, neutral_result)
        _na, _nd = _repair(_neut_call, _NEUT_U, _NEUT_S, neutral_result, _na, _nd, _vn)
        _args["neutral"] = _na
        _arg_drops += _nd
        _nc = "  " + _numbers("Neutral analyst", neutral_result) + "\n" + (_A.forward_text(_na, _AC.cat, _AC.relations) if _na else "")
    elif neutral_result:
        _nc = (
            f"\nNEUTRAL ANALYST VIEW (objective baseline — read before forming your position):\n"
            f"  Return: {neutral_result['expected_return']*100:+.2f}% | "
            f"Recommendation: {neutral_result.get('recommendation','HOLD')} | "
            f"Confidence: {neutral_result['confidence']*100:.0f}%\n"
            f"  Reasoning: {neutral_result.get('reasoning','')}\n"
        )
    else:
        _nc = ""

    # ── Round 2: Bear (Kimi) and Bull (GPT) — each sees neutral view ─────────
    bear_r1 = bull_r1 = None
    print(f"    {DIM}Round 2 — Bear (Risk Mgr) · Bull (Growth Analyst) initial{RESET}")
    pass  # (removed cosmetic inter-round pause)

    _bear_signals = signals_text + _nc
    _bull_signals = signals_text + _nc
    if _AC is not None:
        _BEAR_U, _BEAR_S = _PP.seat_r1("bear", ticker, _brief2, _menu, neutral_block=_nc), _PP.system("bear")
        _BULL_U, _BULL_S = _PP.seat_r1("bull", ticker, _brief2, _menu, neutral_block=_nc), _PP.system("bull")
    else:
        _BEAR_U = _BEAR_USER_TEMPLATE.format(ticker=ticker, months=12, current_price=current_price, signals_text=_bear_signals)
        _BULL_U = _BULL_USER_TEMPLATE.format(ticker=ticker, months=12, current_price=current_price, signals_text=_bull_signals)
        _BEAR_S, _BULL_S = _BEAR_SYSTEM_PROMPT, _BULL_SYSTEM_PROMPT

    # Bear (Kimi, slow) and Bull (GPT) are independent given the neutral view → run concurrently.
    from concurrent.futures import ThreadPoolExecutor
    _ex2 = ThreadPoolExecutor(max_workers=2)
    _fut_bear = _ex2.submit(_call_kimi, _BEAR_U, system=_BEAR_S, verbose=verbose) if has_kimi else None
    _fut_bull = _ex2.submit(_call_openai, _BULL_U, system=_BULL_S, verbose=verbose) if has_openai else None

    bear_model_name = None
    if _fut_bear is not None:
        try:
            bear_r1 = _fut_bear.result()
            if bear_r1:
                bear_model_name = "Kimi"
        except Exception as e:
            print(f"      {'Kimi (Bear)':<20} {RED}{str(e)[:60]}{RESET}")

    # The Bear is the adversarial half of the panel. Kimi (Moonshot) rate-limits hard, and
    # losing it silently turns the debate into a bull-only echo chamber that still reads as
    # "unanimous". Fall back across the other providers so the adversarial structure survives.
    if not bear_r1:
        _bear_fallbacks = ([("Gemini", _call_gemini_3_pro)] if has_gemini else []) \
                        + ([("Claude", _call_claude)] if has_claude else []) \
                        + ([("GPT", _call_openai)] if has_openai else [])
        for _bname, _bcall in _bear_fallbacks:
            print(f"      {'Kimi (Bear)':<20} {YELLOW}unavailable → falling back to {_bname}{RESET}")
            try:
                bear_r1 = _bcall(_BEAR_U, system=_BEAR_S, verbose=verbose)
            except Exception as e:
                print(f"      {f'{_bname} (Bear)':<20} {RED}{str(e)[:60]}{RESET}")
                bear_r1 = None
            if bear_r1:
                bear_model_name = _bname
                break

    if bear_r1:
        total_tokens_in  += bear_r1.get("tokens_in", 0)
        total_tokens_out += bear_r1.get("tokens_out", 0)
        rc = GREEN if bear_r1["expected_return"] > 0 else RED
        _tag = f"{bear_model_name} (Bear)" + ("" if bear_model_name == "Kimi" else " [fallback]")
        print(f"      {_tag:<20} {rc}{bear_r1['expected_return']*100:+.1f}%{RESET} "
              f"{bear_r1.get('recommendation','HOLD'):<11} conf:{bear_r1['confidence']*100:.0f}%")

    if _fut_bull is not None:
        try:
            bull_r1 = _fut_bull.result()
            if bull_r1:
                total_tokens_in  += bull_r1.get("tokens_in", 0)
                total_tokens_out += bull_r1.get("tokens_out", 0)
                rc = GREEN if bull_r1["expected_return"] > 0 else RED
                print(f"      {'GPT (Bull)':<20} {rc}{bull_r1['expected_return']*100:+.1f}%{RESET} "
                      f"{bull_r1.get('recommendation','HOLD'):<11} conf:{bull_r1['confidence']*100:.0f}%")
        except Exception as e:
            print(f"      {'GPT (Bull)':<20} {RED}{str(e)[:60]}{RESET}")
    _ex2.shutdown(wait=True)

    if _AC is not None:
        _open_calls = {"bear": ({"Kimi": _call_kimi, "Gemini": _call_gemini_3_pro, "Claude": _call_claude,
                                 "GPT": _call_openai}.get(bear_model_name), _BEAR_U, _BEAR_S),
                       "bull": (_call_openai, _BULL_U, _BULL_S)}

        def _open(_role, _r):
            _v = lambda rep: _A.validate_seat(_AC, _role, rep, own=_own_r(_r["expected_return"]))
            _sa, _sd = _settle(_v, _r)
            _c, _u, _s = _open_calls[_role]
            return _repair(_c, _u, _s, _r, _sa, _sd, _v)
        _ex_o = ThreadPoolExecutor(max_workers=2)
        _fo = {_role: _ex_o.submit(_open, _role, _r) for _role, _r in (("bear", bear_r1), ("bull", bull_r1)) if _r}
        for _role, _f in _fo.items():
            _sa, _sd = _f.result()
            _args[f"{_role}_open"] = _sa
            _arg_drops += _sd
        _ex_o.shutdown(wait=True)

    # ── Round 3: Bear and Bull debate each other ──────────────────────────────
    bear_final = bear_r1
    bull_final = bull_r1
    _rebut_calls = {}
    print(f"    {DIM}Round 3 — Bear vs Bull debate{RESET}")
    pass  # (removed cosmetic inter-round pause)

    _nr      = neutral_result or {}
    _nr_ret  = _nr.get("expected_return", 0.0) * 100
    _nr_rec  = _nr.get("recommendation", "N/A")
    _nr_conf = _nr.get("confidence", 0.5) * 100
    _nr_rsn  = _nr.get("reasoning", "not available")

    # Gate: only run the rebuttal round when Bull and Bear actually disagree materially
    # (saves 2 LLM calls when the advocates already align); run both concurrently otherwise.
    if bear_r1 and bull_r1 and abs(bull_r1["expected_return"] - bear_r1["expected_return"]) >= 0.08:
        _br1_price = bear_r1.get("predicted_price") or current_price * (1 + bear_r1["expected_return"])
        _bl1_price = bull_r1.get("predicted_price") or current_price * (1 + bull_r1["expected_return"])
        _ex3 = ThreadPoolExecutor(max_workers=2)
        _bear_rebut_call = {"Kimi": _call_kimi, "Gemini": _call_gemini_3_pro,
                            "Claude": _call_claude, "GPT": _call_openai}.get(bear_model_name)
        if _AC is not None:
            _BR_U = _PP.rebuttal("bear", "bull", ticker, _menu,
                                 {"label": _rating_for(round(float(bear_r1["expected_return"]), 3)),
                                  "return_pct": bear_r1["expected_return"] * 100},
                                 (_args.get("bear_open") or {}).get("thesis"),
                                 _A.forward_text(_args.get("bull_open"), _AC.cat, _AC.relations, indexed=True))
            _BL_U = _PP.rebuttal("bull", "bear", ticker, _menu,
                                 {"label": _rating_for(round(float(bull_r1["expected_return"]), 3)),
                                  "return_pct": bull_r1["expected_return"] * 100},
                                 (_args.get("bull_open") or {}).get("thesis"),
                                 _A.forward_text(_args.get("bear_open"), _AC.cat, _AC.relations, indexed=True))
            _rebut_calls["bear"] = (_bear_rebut_call or _call_kimi, _BR_U, _PP.system("bear"))
            _rebut_calls["bull"] = (_call_openai, _BL_U, _PP.system("bull"))
            _fut_br = _ex3.submit(_bear_rebut_call or _call_kimi, _BR_U, system=_PP.system("bear"),
                                  verbose=verbose) if bear_model_name else None
            _fut_bl = _ex3.submit(_call_openai, _BL_U, system=_PP.system("bull"), verbose=verbose) if has_openai else None
        else:
          _fut_br = _ex3.submit(
            _bear_rebut_call or _call_kimi,
            _BEAR_REBUTTAL_TEMPLATE.format(
                ticker=ticker, months=12, current_price=current_price,
                neutral_return=_nr_ret, neutral_rec=_nr_rec,
                neutral_conf=_nr_conf, neutral_reasoning=_nr_rsn,
                my_return=bear_r1["expected_return"] * 100, my_price=_br1_price,
                my_rec=bear_r1.get("recommendation", "HOLD"), my_reasoning=bear_r1.get("reasoning", ""),
                bull_return=bull_r1["expected_return"] * 100, bull_price=_bl1_price,
                bull_rec=bull_r1.get("recommendation", "HOLD"), bull_conf=bull_r1["confidence"] * 100,
                bull_reasoning=bull_r1.get("reasoning", ""),
                bull_catalysts=", ".join((bull_r1.get("risk_flags") or [])[:4]),
            ),
            system=_BEAR_SYSTEM_PROMPT, verbose=verbose) if bear_model_name else None
          _fut_bl = _ex3.submit(
            _call_openai,
            _BULL_REBUTTAL_TEMPLATE.format(
                ticker=ticker, months=12, current_price=current_price,
                neutral_return=_nr_ret, neutral_rec=_nr_rec,
                neutral_conf=_nr_conf, neutral_reasoning=_nr_rsn,
                my_return=bull_r1["expected_return"] * 100, my_price=_bl1_price,
                my_rec=bull_r1.get("recommendation", "HOLD"), my_reasoning=bull_r1.get("reasoning", ""),
                bear_return=bear_r1["expected_return"] * 100, bear_price=_br1_price,
                bear_rec=bear_r1.get("recommendation", "HOLD"), bear_conf=bear_r1["confidence"] * 100,
                bear_reasoning=bear_r1.get("reasoning", ""),
                bear_risks=", ".join((bear_r1.get("risk_flags") or [])[:4]),
            ),
            system=_BULL_SYSTEM_PROMPT, verbose=verbose) if has_openai else None

        if _fut_br is not None:
            try:
                bear_rebuttal = _fut_br.result()
                if bear_rebuttal and _AC is not None and not bear_rebuttal.get("has_return"):
                    _arg_drops.append({"source": "bear.rebuttal", "raw": None,
                                       "reason": "REBUTTAL_NO_NUMBERS: the reply had no expected return; round 1 stands"})
                    bear_rebuttal = None
                if bear_rebuttal:
                    bear_final = bear_rebuttal
                    total_tokens_in  += bear_rebuttal.get("tokens_in", 0)
                    total_tokens_out += bear_rebuttal.get("tokens_out", 0)
                    changed = bear_rebuttal.get("changed_view", False)
                    rc  = GREEN if bear_rebuttal["expected_return"] > 0 else RED
                    tag = f"{YELLOW}REVISED{RESET}" if changed else f"{DIM}held{RESET}"
                    print(f"      {'Kimi (Bear)':<20} {rc}{bear_rebuttal['expected_return']*100:+.1f}%{RESET} "
                          f"{bear_rebuttal.get('recommendation','HOLD'):<11} {tag}")
            except Exception as e:
                print(f"      {'Kimi (Bear)':<20} {RED}{str(e)[:60]}{RESET}")

        if _fut_bl is not None:
            try:
                bull_rebuttal = _fut_bl.result()
                if bull_rebuttal and _AC is not None and not bull_rebuttal.get("has_return"):
                    _arg_drops.append({"source": "bull.rebuttal", "raw": None,
                                       "reason": "REBUTTAL_NO_NUMBERS: the reply had no expected return; round 1 stands"})
                    bull_rebuttal = None
                if bull_rebuttal:
                    bull_final = bull_rebuttal
                    total_tokens_in  += bull_rebuttal.get("tokens_in", 0)
                    total_tokens_out += bull_rebuttal.get("tokens_out", 0)
                    changed = bull_rebuttal.get("changed_view", False)
                    rc  = GREEN if bull_rebuttal["expected_return"] > 0 else RED
                    tag = f"{YELLOW}REVISED{RESET}" if changed else f"{DIM}held{RESET}"
                    print(f"      {'GPT (Bull)':<20} {rc}{bull_rebuttal['expected_return']*100:+.1f}%{RESET} "
                          f"{bull_rebuttal.get('recommendation','HOLD'):<11} {tag}")
            except Exception as e:
                print(f"      {'GPT (Bull)':<20} {RED}{str(e)[:60]}{RESET}")
        _ex3.shutdown(wait=True)
    elif bear_r1 and bull_r1:
        print(f"      {DIM}Bull & Bear within "
              f"{abs(bull_r1['expected_return']-bear_r1['expected_return'])*100:.1f}% — rebuttals skipped (aligned){RESET}")

    _bear_rebutted = bear_final is not bear_r1
    _bull_rebutted = bull_final is not bull_r1
    if _AC is not None:
        def _rebut(_role, _opp, _fin, _r1):
            _v = lambda rep: _A.validate_rebuttal(_AC, _role, rep, own=_own_r(_fin["expected_return"]),
                                                  opponent=_args.get(f"{_opp}_open"), opponent_role=f"{_opp}_open",
                                                  r1_value=_r1["expected_return"],
                                                  r1_label=_rating_for(round(float(_r1["expected_return"]), 3)),
                                                  opening_thesis=(_args.get(f"{_role}_open") or {}).get("thesis"))
            _ra, _rd = _settle(_v, _fin)
            _c, _u, _s = _rebut_calls.get(_role, (None, None, None))
            return _repair(_c, _u, _s, _fin, _ra, _rd, _v)
        _ex_r = ThreadPoolExecutor(max_workers=2)
        _fr = {_role: _ex_r.submit(_rebut, _role, _opp, _fin, _r1)
               for _role, _opp, _fin, _r1, _did in (("bear", "bull", bear_final, bear_r1, _bear_rebutted),
                                                     ("bull", "bear", bull_final, bull_r1, _bull_rebutted)) if _did and _fin}
        for _role, _f in _fr.items():
            _ra, _rd = _f.result()
            _args[_role] = _ra
            _arg_drops += _rd
        _ex_r.shutdown(wait=True)

    # ── Round 4: Portfolio Manager synthesis ──────────────────────────────────
    pm_synthesis = None
    print(f"    {DIM}Round 4 — Portfolio Manager synthesis{RESET}")
    pass  # (removed cosmetic inter-round pause)

    if has_claude and (bear_final or bull_final):
        _br = bear_final or {"expected_return": 0.0, "predicted_price": current_price,
                              "recommendation": "HOLD", "confidence": 0.5,
                              "reasoning": "unavailable", "risk_flags": []}
        _bl = bull_final or {"expected_return": 0.0, "predicted_price": current_price,
                              "recommendation": "HOLD", "confidence": 0.5,
                              "reasoning": "unavailable", "risk_flags": []}
        _br_price = _br.get("predicted_price") or current_price * (1 + _br["expected_return"])
        _bl_price = _bl.get("predicted_price") or current_price * (1 + _bl["expected_return"])
        try:
          if _AC is not None:
            # the semantic gate runs BEFORE the portfolio manager weighs the scenarios (it used to run after the state
            # froze, so the PM weighed arguments the report then removed): the PM reads only surviving arguments and
            # is told each seat's evidence state. Nothing is zeroed, capped or renormalized here.
            _gate_drops = []
            _pre_gate = _sm.get("pre_pm_gate")
            if callable(_pre_gate):
                try:
                    _gated, _gate_drops = _pre_gate({_r: _args[_r] for _r in _A.SEAT_ROLES if _args.get(_r)})
                    _args.update(_gated)
                except Exception as _ge:  # noqa: BLE001 — the compose-time gate still runs on ungated sets
                    print(f"      {YELLOW}pre-PM semantic gate unavailable ({type(_ge).__name__}: {str(_ge)[:80]}){RESET}")
            _ev_state = _A.seat_evidence_state(_args, list(_arg_drops) + list(_gate_drops))
            _dbg = sorted(_A.debate_evidence(_args))
            _blk = {}
            if neutral_result:
                _blk["neutral"] = "NEUTRAL ANALYST — " + _numbers("return", neutral_result) + "\n" + \
                    _A.forward_text(_args.get("neutral"), _AC.cat, _AC.relations) + \
                    "\n" + _A.evidence_state_line("neutral", _ev_state["neutral"])
            for _role, _fin, _did in (("bear", bear_final, _bear_rebutted), ("bull", bull_final, _bull_rebutted)):
                if not _fin:
                    continue
                _t = f"{_role.upper()} — " + _numbers("final return", _fin) + "\n" + \
                    _A.forward_text(_args.get(f"{_role}_open"), _AC.cat, _AC.relations, indexed=True)
                _rb = _args.get(_role) if _did else None
                if _rb:
                    _t += "\n  Rebuttal: " + (_rb.get("thesis") or "") + "".join(
                        f"\n   - on {x['to']} ({_A.VERDICT_CHIP[x['verdict']].lower()}): {x['view']}" for x in _rb.get("answers") or [])
                _blk[_role] = _t + "\n" + _A.evidence_state_line(_role, _ev_state[_role])
            _blk["ids"] = ("FACTORS may cite ONLY these IDs (the evidence the validated debate used): "
                           + (", ".join(_dbg) or "(none)"))
            _PM_U = _PP.pm(
                ticker, _sm["pm_anchors"], _sm["panel_line"], _blk, _menu,
                ", ".join(o["id"] for o in (_sm.get("evidence_offers") or []) if o["eligible"]) or "intrinsic_value, market_pricing",
                "; ".join(f"{o['id']} ({o.get('reason') or 'not available'})"
                          for o in (_sm.get("evidence_offers") or []) if not o["eligible"]) or "none")
            pm_synthesis = _call_claude_synthesis(_PM_U, verbose=verbose)
          else:
            pm_synthesis = _call_claude_synthesis(
                _PM_SYNTHESIS_USER_TEMPLATE_YEARLY.format(
                    ticker=ticker, current_price=current_price, months=12,
                    panel_ml=_sm["panel_line"],
                    anchors=_sm["pm_anchors"],
                    evidence_ids=", ".join(o["id"] for o in (_sm.get("evidence_offers") or []) if o["eligible"])
                                 or "intrinsic_value, market_pricing",
                    evidence_ineligible="; ".join(f"{o['id']} ({o.get('reason') or 'not available'})"
                                                  for o in (_sm.get("evidence_offers") or []) if not o["eligible"]) or "none",
                    neutral_return=_nr_ret, neutral_rec=_nr_rec,
                    neutral_conf=_nr_conf, neutral_reasoning=_nr_rsn,
                    bear_return=_br["expected_return"] * 100,
                    bear_price=_br_price,
                    bear_rec=_br.get("recommendation", "HOLD"),
                    bear_conf=_br["confidence"] * 100,
                    bear_reasoning=_br["reasoning"],
                    bear_risks=", ".join((_br.get("risk_flags") or [])[:4]),
                    bull_return=_bl["expected_return"] * 100,
                    bull_price=_bl_price,
                    bull_rec=_bl.get("recommendation", "HOLD"),
                    bull_conf=_bl["confidence"] * 100,
                    bull_reasoning=_bl["reasoning"],
                    bull_catalysts=", ".join((_bl.get("risk_flags") or [])[:4]),
                ),
                verbose=verbose,
            )
          if pm_synthesis:
                total_tokens_in  += pm_synthesis.get("tokens_in", 0)
                total_tokens_out += pm_synthesis.get("tokens_out", 0)
        except Exception as e:
            print(f"      {'Claude (PM)':<20} {RED}synthesis failed: {str(e)[:60]}{RESET}")

    # ── Compute final return mechanically ────────────────────────────────────
    _bear_ret = (bear_final or {}).get("expected_return")
    _bull_ret = (bull_final or {}).get("expected_return")

    if pm_synthesis and _bear_ret is not None and _bull_ret is not None:
        p_bear = pm_synthesis["p_bear"]
        p_bull = pm_synthesis["p_bull"]
        if p_bear + p_bull > 0.95:
            scale  = 0.95 / (p_bear + p_bull)
            p_bear *= scale
            p_bull *= scale
        p_neutral        = max(1.0 - p_bear - p_bull, 0.05)
        # §15 — the odds are the panel manager's judgement; any quantitative tilt is recorded
        # separately and bounded by that model's reliability, so an UNRELIABLE model contributes
        # exactly zero rather than leaking in through the manager's prose.
        try:
            import concepts as _ccp
            _sprob = _ccp.scenario_probabilities(
                {"bear": p_bear, "neutral": p_neutral, "bull": p_bull},
                (data.get("quant_reliability") if isinstance(data, dict) else None),
                quant_adjustment=(pm_synthesis.get("quant_tilt") if isinstance(pm_synthesis.get("quant_tilt"), dict) else None))
            _sp_audit = _sprob.audit()
            _fin = _sp_audit["final_probability"]
            p_bear, p_neutral, p_bull = _fin["bear"], _fin["neutral"], _fin["bull"]
        except Exception as _spe:  # noqa: BLE001
            _sp_audit = {"error": str(_spe)[:80]}
        try:
            import concepts as _cce
            _qrl_e = (data.get("quant_reliability") if isinstance(data, dict) else None) or {}
            _led = _cce.EvidenceLedger()
            # every item's eligibility comes from the validated state (brief.evidence_offers), not from this prompt
            for _o in (_sm.get("evidence_offers") or []):
                _led.offer(_o["id"], _o["description"], eligible=_o["eligible"], reason=_o.get("reason"))
            for _eid in (pm_synthesis.get("evidence_used") or []):
                _led.cite("recommendation", _eid)
            _ev_audit = _led.audit()
        except Exception as _eve:  # noqa: BLE001
            _ev_audit = {"error": str(_eve)[:80]}
        # Proper 3-scenario expected value — the neutral leg was previously dropped,
        # so ~30% of the probability mass contributed nothing (biased the number toward 0).
        _neut_ret        = float((neutral_result or {}).get("expected_return", 0.0) or 0.0)
        final_return     = p_bear * _bear_ret + p_bull * _bull_ret + p_neutral * _neut_ret
        final_confidence = pm_synthesis["conviction"]
        pm_reasoning     = pm_synthesis["reasoning"]
        all_flags        = _dedup_flags(
            (bear_final or {}).get("risk_flags", []) + pm_synthesis.get("risk_flags", [])
        )
        print(f"      {'Claude (PM)':<20} "
              f"p_bear={p_bear:.0%}  p_bull={p_bull:.0%}  p_neutral={p_neutral:.0%}  "
              f"→ {GREEN if final_return > 0 else RED}{final_return*100:+.1f}%{RESET}")
    elif _bear_ret is not None and _bull_ret is not None:
        # PM call failed — degrade gracefully: weight each side by its OWN stated
        # confidence (not a flat 40/40) and keep a neutral anchor, so the number
        # still reflects who argued more convincingly.
        _bc = float((bear_final or {}).get("confidence", 0.5))
        _lc = float((bull_final or {}).get("confidence", 0.5))
        _tot = (_bc + _lc + 0.5) or 1.0
        p_bear    = float(np.clip(_bc / _tot, 0.10, 0.65))
        p_bull    = float(np.clip(_lc / _tot, 0.10, 0.65))
        p_neutral = max(1.0 - p_bear - p_bull, 0.05)
        _neut_ret = float((neutral_result or {}).get("expected_return", 0.0) or 0.0)
        final_return     = p_bear * _bear_ret + p_bull * _bull_ret + p_neutral * _neut_ret
        final_confidence = float(np.mean([_bc, _lc]))
        pm_reasoning = (f"PM synthesis unavailable — confidence-weighted blend "
                        f"(bear {p_bear:.0%} @ {_bear_ret*100:+.1f}%, bull {p_bull:.0%} @ {_bull_ret*100:+.1f}%).")
        all_flags    = _dedup_flags(
            (bear_final or {}).get("risk_flags", []) + (bull_final or {}).get("risk_flags", [])
        )
    # ── One-sided panel (an advocate's provider failed) ──────────────────────────
    # Adopting the surviving advocate wholesale would publish an UNCHALLENGED case — a
    # bull-only panel reading as a high-conviction BUY purely because the bear's API was
    # down. Anchor to the neutral baseline, halve confidence, and flag it.
    elif _bear_ret is not None:
        p_bear = 0.5; p_bull = 0.0; p_neutral = 0.5
        _nb = (neutral_result or {}).get("expected_return")
        final_return     = _bear_ret if _nb is None else (0.5 * _bear_ret + 0.5 * _nb)
        final_confidence = min((bear_final or {}).get("confidence", 0.5) * 0.5, 0.35)
        pm_reasoning     = ("Bull analyst unavailable (provider error) — ONE-SIDED panel: the bear "
                            "case was never challenged, so it is anchored to the neutral baseline "
                            "and confidence is reduced.")
        all_flags        = list((bear_final or {}).get("risk_flags", [])) + ["ONE_SIDED_PANEL: bull analyst unavailable"]
    elif _bull_ret is not None:
        p_bear = 0.0; p_bull = 0.5; p_neutral = 0.5
        _nb = (neutral_result or {}).get("expected_return")
        final_return     = _bull_ret if _nb is None else (0.5 * _bull_ret + 0.5 * _nb)
        final_confidence = min((bull_final or {}).get("confidence", 0.5) * 0.5, 0.35)
        pm_reasoning     = ("Bear analyst unavailable (provider error) — ONE-SIDED panel: the bull "
                            "case was never challenged, so it is anchored to the neutral baseline "
                            "and confidence is reduced.")
        all_flags        = list((bull_final or {}).get("risk_flags", [])) + ["ONE_SIDED_PANEL: bear analyst unavailable"]
    elif neutral_result:
        p_bear = p_bull = 0.0; p_neutral = 1.0
        final_return     = neutral_result["expected_return"]
        final_confidence = neutral_result["confidence"]
        pm_reasoning     = neutral_result.get("reasoning", "")
        all_flags        = neutral_result.get("risk_flags", [])
    else:
        return None

    if _AC is not None and pm_synthesis:
        _vp = lambda rep: _A.validate_pm(_AC, rep, own=_own_r(final_return), debate_ids=_A.debate_evidence(_args))
        _pa, _pd = _settle(_vp, pm_synthesis)
        _pa, _pd = _repair(_call_claude_synthesis, locals().get("_PM_U"), None, pm_synthesis, _pa, _pd, _vp)
        _args["pm"] = _pa
        _arg_drops += _pd

    # Direction and recommendation from the mechanical return
    if final_return > 0.005:
        final_direction = "BULLISH"
    elif final_return < -0.005:
        final_direction = "BEARISH"
    else:
        final_direction = "NEUTRAL"

    # Same bands as the app's headline rating (BUY ≥ +10%, HOLD −5…+10%, SELL < −5%), so a
    # +6% odds-weighted return reads HOLD — the PM's prose said "HOLD-like" while the label said BUY.
    if final_return >= 0.20:
        final_rec = "STRONG BUY"
    elif final_return >= 0.10:
        final_rec = "BUY"
    elif final_return >= -0.05:
        final_rec = "HOLD"
    elif final_return >= -0.20:
        final_rec = "SELL"
    else:
        final_rec = "STRONG SELL"

    final_predicted_price = current_price * (1 + final_return)

    # ── ONE scenario object: the renderer, the PM card and the Stage B validator all read this ──
    import concepts as _cc_s
    _scen = _cc_s.scenario_set_from(p_bear, p_neutral, p_bull,
                                    (bear_final or bear_r1 or {}).get("expected_return"),
                                    (neutral_result or {}).get("expected_return"),
                                    (bull_final or bull_r1 or {}).get("expected_return"),
                                    final_return)
    _scen_status, _scen_detail, _scen_skip = _scen.validate()
    print(f"      scenarios: {_scen_status} — {_scen_detail}")
    # ── structured thesis triggers (metric, direction, threshold, destination) with causal polarity ──
    _triggers = _cc_s.build_triggers((pm_synthesis or {}).get("triggers"))
    _trig_prose = "; ".join(t.sentence() for t in _triggers) or None
    if not _triggers and (pm_synthesis or {}).get("tipping_point"):
        print("      triggers: none structured — the flip condition stays free text and is labelled unstructured")
    if any(t.status == "POLARITY_REVERSED" for t in _triggers):
        print(f"      {YELLOW}trigger polarity corrected: " + "; ".join(f"{t.metric_id} → {t.scenario_destination}" for t in _triggers if t.status == "POLARITY_REVERSED") + RESET)

    # Display-compatible round dicts
    # Enforce the 12-month label bands in code: the personas mostly comply with the rubric
    # but not always (a bear printed STRONG SELL at -12.5%); the UI must never show a label
    # that contradicts the number beside it. Label = f(return), same bands as the prompt.
    def _enforce_band(entry, side):
        if not entry or entry.get("expected_return") is None:
            return entry
        r = float(entry["expected_return"])
        if side == "bull":
            rec = "STRONG BUY" if r >= 0.25 else "BUY" if r >= 0.10 else "HOLD"
        else:
            rec = "STRONG SELL" if r <= -0.15 else "SELL" if r <= -0.05 else "HOLD"
        if entry.get("recommendation") != rec:
            entry = dict(entry); entry["recommendation"] = rec
        return entry
    bear_r1, bull_r1 = _enforce_band(bear_r1, "bear"), _enforce_band(bull_r1, "bull")
    bear_final, bull_final = _enforce_band(bear_final, "bear"), _enforce_band(bull_final, "bull")
    _round1 = {}
    if neutral_result:
        _round1["gemini"] = {
            "model": "Gemini (Neutral)", "return": neutral_result["expected_return"],
            "direction": neutral_result["direction"], "confidence": neutral_result["confidence"],
            "recommendation": neutral_result.get("recommendation", "HOLD"),
            "reasoning": neutral_result.get("reasoning", ""),
            "key_points": neutral_result.get("key_points", []),
        }
    if bear_r1:
        _round1["kimi"] = {
            "model": "Kimi (Bear / Risk Mgr)", "return": bear_r1["expected_return"],
            "direction": bear_r1.get("direction", "BEARISH"), "confidence": bear_r1["confidence"],
            "recommendation": bear_r1.get("recommendation", "HOLD"),
            "reasoning": bear_r1.get("reasoning", ""),
            "key_points": bear_r1.get("key_points", []),
        }
    if bull_r1:
        _round1["gpt"] = {
            "model": "GPT (Bull / Growth)", "return": bull_r1["expected_return"],
            "direction": bull_r1.get("direction", "BULLISH"), "confidence": bull_r1["confidence"],
            "recommendation": bull_r1.get("recommendation", "HOLD"),
            "reasoning": bull_r1.get("reasoning", ""),
            "key_points": bull_r1.get("key_points", []),
        }

    _round2 = {}
    if bear_final:
        _round2["kimi"] = {
            "model": "Kimi (Bear / Risk Mgr)", "return": bear_final["expected_return"],
            "direction": bear_final.get("direction", "BEARISH"), "confidence": bear_final["confidence"],
            "recommendation": bear_final.get("recommendation", "HOLD"),
            "reasoning": bear_final.get("reasoning", ""),
            "key_points": bear_final.get("key_points", []),
            "changed": bear_final.get("changed_view", False),
        }
    if bull_final:
        _round2["gpt"] = {
            "model": "GPT (Bull / Growth)", "return": bull_final["expected_return"],
            "direction": bull_final.get("direction", "BULLISH"), "confidence": bull_final["confidence"],
            "recommendation": bull_final.get("recommendation", "HOLD"),
            "reasoning": bull_final.get("reasoning", ""),
            "key_points": bull_final.get("key_points", []),
            "changed": bull_final.get("changed_view", False),
        }
    _round2["claude"] = {
        "model": "Claude (PM, synthesis)", "return": final_return,
        "direction": final_direction, "confidence": final_confidence,
        "recommendation": final_rec, "reasoning": pm_reasoning,
        "key_points": (pm_synthesis or {}).get("key_points", []),
        "tipping_point": (pm_synthesis or {}).get("tipping_point"),
        "changed": pm_synthesis is not None,
    }

    # structured arguments ride on each round (only validated objects; a reply that ignored the schema keeps its legacy
    # prose path, and the rebuttal slot is empty when no rebuttal happened)
    if _AC is not None:
        _fmt = {}
        for who, role, raw in (("gemini", "neutral", neutral_result), ("kimi", "bear_open", bear_r1), ("gpt", "bull_open", bull_r1)):
            if who in _round1:
                _fmt[role] = "v2" if _A.is_structured(raw) else "legacy"
                if _fmt[role] == "v2":
                    _a = _args.get(role) or {"schema": _A.SCHEMA, "role": role, "format": "v2", "thesis": None, "points": []}
                    _round1[who].update(arguments=_a, reasoning=_a.get("thesis") or "", key_points=[])
        for who, role, raw, did in (("kimi", "bear", bear_final, _bear_rebutted), ("gpt", "bull", bull_final, _bull_rebutted)):
            if who in _round2:
                _round2[who]["rebutted"] = did
                if not did:
                    # the final seat IS the opening case: no second copy of its prose
                    _round2[who].update(arguments=None, reasoning="", key_points=[])
                elif _A.is_structured(raw):
                    _fmt[role] = "v2"
                    _a = _args.get(role) or {"schema": _A.SCHEMA, "role": role, "format": "v2", "thesis": None, "answers": []}
                    _round2[who].update(arguments=_a, reasoning=_a.get("thesis") or "", key_points=[])
                else:
                    _fmt[role] = "legacy"
        if pm_synthesis is not None:
            _fmt["pm"] = "v2" if _A.is_structured(pm_synthesis) else "legacy"
            if _fmt["pm"] == "v2":
                _a = _args.get("pm") or {"schema": _A.SCHEMA, "role": "pm", "format": "v2", "thesis": None, "points": []}
                _round2["claude"].update(arguments=_a, reasoning=_a.get("thesis") or "", key_points=[])
        print(f"      {DIM}structured arguments: {_fmt}; {len(_arg_drops)} object(s) dropped in the debate{RESET}")

    # Prose fields
    _br_lbl = "Kimi (Bear)"
    _bl_lbl = "GPT (Bull)"
    _br_v   = bear_final or {}
    _bl_v   = bull_final or {}
    summary_parts = []
    if bear_final:
        summary_parts.append(
            f"**{_br_lbl}** ({bear_final['expected_return']*100:+.1f}%, "
            f"{bear_final.get('recommendation','HOLD')}): {bear_final.get('reasoning','')}"
        )
    if bull_final:
        summary_parts.append(
            f"**{_bl_lbl}** ({bull_final['expected_return']*100:+.1f}%, "
            f"{bull_final.get('recommendation','HOLD')}): {bull_final.get('reasoning','')}"
        )
    summary_parts.append(
        f"**Claude (PM)** ({final_return*100:+.1f}%, {final_rec}): {pm_reasoning}"
    )

    result = {
        "expected_return":  final_return,
        "predicted_price":  final_predicted_price,
        "recommendation":   final_rec,
        "confidence":       float(np.clip(final_confidence, 0.1, 1.0)),
        "direction":        final_direction,
        "reasoning":        pm_reasoning,
        "risk_flags":       all_flags[:8],
        "risk_factors":     all_flags[:8],
        "override_ml":      any(bool((r or {}).get("override_ml")) for r in
                                [neutral_result, bear_r1, bull_r1, pm_synthesis]),
        "model":            "persona-debate",
        "tokens_in":        total_tokens_in,
        "tokens_out":       total_tokens_out,
        "scenarios":        _scen.as_dict(),
        "scenario_object_id": _scen.object_id,
        "thesis_triggers":  [t.as_dict() for t in _triggers],
        # what the PM was told about each seat's arguments when it set the odds (None on the legacy prose path)
        "pm_evidence_state": locals().get("_ev_state"),
        "tipping_point_rendered": _trig_prose,
        "p_bear":           p_bear,
        "p_bull":           p_bull,
        "p_neutral":        p_neutral,
        "scenario_probability_audit": _sp_audit,
        "evidence_lineage": _ev_audit,
        "rationale":        "  ·  ".join(summary_parts),
        "bull_case":        (f"**{_bl_lbl}** ({_bl_v.get('expected_return',0)*100:+.1f}%, "
                             f"{_bl_v.get('recommendation','HOLD')}): {_bl_v.get('reasoning','')}"),
        "bear_case":        (f"**{_br_lbl}** ({_br_v.get('expected_return',0)*100:+.1f}%, "
                             f"{_br_v.get('recommendation','HOLD')}): {_br_v.get('reasoning','')}"),
        "round1":           _round1,
        "round2":           _round2,
        "argument_drops":   [dict(d, raw=str(d.get("raw"))[:300]) for d in _arg_drops],
        # A panel missing an advocate is NOT unanimous — that would imply the absent
        # analyst agreed. Report the incomplete state honestly.
        "panel_complete":   bool(_bear_ret is not None and _bull_ret is not None),
        "bear_model":       bear_model_name,
        "vote":             {"direction": final_direction, "count": 1, "total": 1,
                             "unanimous": bool(_bear_ret is not None and _bull_ret is not None)},
        "rec_vote":         {"recommendation": final_rec, "count": 1, "total": 1,
                             "unanimous": bool(_bear_ret is not None and _bull_ret is not None)},
        "vote_summary":     ("Four-seat panel complete — odds-weighted verdict, not a unanimous vote" if (_bear_ret is not None and _bull_ret is not None)
                             else ("Incomplete panel — bear analyst unavailable" if _bear_ret is None
                                   else "Incomplete panel — bull analyst unavailable")),
        "debate_summary":   (
            f"Bear: {(_bear_ret or 0)*100:+.1f}%  |  Bull: {(_bull_ret or 0)*100:+.1f}%  |  "
            f"PM weights: p_bear={p_bear:.0%}  p_bull={p_bull:.0%}  p_neutral={p_neutral:.0%}"
            f"  →  {final_return*100:+.1f}%  {final_rec}"
        ),
    }

    print(f"    → Final: {GREEN if final_return > 0 else RED}{final_return*100:+.2f}%{RESET} "
          f"→ ${final_predicted_price:.2f} | {final_rec} | conf:{final_confidence*100:.0f}%")
    if result["override_ml"]:
        print(f"    {YELLOW}⚠ Panel flagged the supplementary price model as unreliable{RESET}")

    return result




_TEXT_AUDIT_SYSTEM = """You are the fact and arithmetic checker for an equity-research debate. You get the
CANONICAL NUMBERS and the analysts' statements. Flag ONLY statements that are objectively wrong:
 (a) an arithmetic result that does not follow from the figures cited (e.g. "$20.06 x 20.2 = $365.71");
 (b) a number attributed to the wrong quantity (calling the market composite a DCF or an intrinsic value,
     calling the story value a multiple-based price, using the forward P/E to "derive" the composite);
 (c) a deal described as completed when the canonical facts say it is pending, or a divested business
     described as a current contributor;
 (d) a stated figure that contradicts the canonical numbers by more than rounding;
 (f) a growth comparison that pairs the continuing-company forecast with the pre-divestiture trailing
     figure and calls it a decline/compression, or a forward multiple set against a trailing one;
 (g) return arithmetic: a 12-month return claim must equal roughly EPS growth + multiple change + yield;
     "the multiple stays the same" with 9% EPS growth cannot produce a 20% return — fix the number or the claim;
 (h) a 13F / insider / super-investor count that contradicts the canonical counts, or one analyst's count
     that contradicts another's when the canonical figure settles it;
 (i) an inference of management's motives or private view from insider selling — restate as "net insider
     selling is a negative factor signal";
 (j) the engine's forecast margin/EPS/revenue described as guidance or consensus — restate as "the engine forecasts";
 (k) a profit/EBITDA growth claim inconsistent with the revenue growth and margin change cited in the same
     statement (operating profit growth ≈ (1+revenue growth)×(new margin/old margin) − 1);
 (l) an organizational or segment placement asserted for a PENDING deal (which unit it "augments" or reports
     into) that is not in the canonical facts — prefix "(unverified) ";
 (e) a SPECIFIC historical corporate claim (a past deal's cost or synergy outcome, a named regulator's
     active probe, a lawsuit result, an integration figure) that is not in the canonical numbers or the
     KNOWN CORPORATE FACTS — for these, keep the sentence but prefix the claim with "(unverified) ".
Do NOT flag opinions, forecasts, or stylistic issues. For each flag return the EXACT substring to replace
(copied verbatim from the statement, 20-220 characters) and a replacement of similar length that keeps
the analyst's stance but is correct. Max 8 fixes.
STRICT JSON only: {"fixes": [{"who": "gemini|gpt|kimi|claude", "round": "round1|round2",
                             "find": "<verbatim substring>", "replace": "<corrected>", "issue": "<=12 words"}]}"""




# ─── Blend (Yearly weights) ─────────────────────────────────────────────────


def blend_predictions_yearly(fund_ml_val, macro, ml, llm=None, scanner=None):
    """Confidence-weighted blend: FV / Macro / ML; with --llm, LLM Debate is a first-class source."""

    macro_adj = dict(macro) if macro else None
    # audit trail: raw inputs before any cap / dampening / penalty, so the headline is reproducible
    _raw_ret = {"Multi-Method Fair Value": (fund_ml_val or {}).get("expected_return"), "Macro": (macro or {}).get("expected_return"),
                "Price Model (suppl.)": (ml or {}).get("expected_return"), "Analyst Panel": (llm or {}).get("expected_return")}
    _raw_conf = {"Multi-Method Fair Value": (fund_ml_val or {}).get("confidence"), "Macro": (macro or {}).get("confidence"),
                 "Price Model (suppl.)": (ml or {}).get("confidence"), "Analyst Panel": (llm or {}).get("confidence")}
    _cap_note = {}
    if macro_adj:
        macro_return = abs(macro_adj.get("expected_return", 0))
        if macro_return < 0.01:
            macro_adj["confidence"] = macro_adj["confidence"] * 0.10
        elif macro_return < 0.03:
            macro_adj["confidence"] = macro_adj["confidence"] * 0.40
        elif macro_return < 0.05:
            macro_adj["confidence"] = macro_adj["confidence"] * 0.60

    fund_src = dict(fund_ml_val) if fund_ml_val else None
    ml_adj = dict(ml) if ml else None
    llm_adj = dict(llm) if llm else None

    # FV magnitude sanity cap: single-year FV returns above ±50% are suspect
    if fund_src:
        _fv_ret_cap = float(fund_src.get("expected_return", 0) or 0)
        if abs(_fv_ret_cap) > 0.50:
            fund_src["expected_return"] = float(np.sign(_fv_ret_cap) * 0.50)
            fund_src["fv_magnitude_capped"] = True
            fund_src["confidence"] = float(fund_src.get("confidence", 0.5) or 0) * 0.70
            _cap_note["Multi-Method Fair Value"] = f"implied return capped at ±50% (raw {_fv_ret_cap*100:+.0f}%); confidence ×0.70"

    _qr_cap = None
    if ml_adj and isinstance(ml_adj.get("_reliability"), dict):
        _qr_cap = float(ml_adj["_reliability"].get("weight_cap", 1.0))
    if ml_adj:
        ml_dir_acc_b = float(ml_adj.get("backtest_dir_accuracy", 0.5) or 0.5)
        ml_ret0 = float(ml_adj.get("expected_return", 0))
        if ml_dir_acc_b < 0.55:
            magnitude_scale = max(0.20, (ml_dir_acc_b - 0.45) / 0.10)
            ml_adj["expected_return"] = ml_ret0 * magnitude_scale
            ml_adj["magnitude_dampen_scale"] = float(magnitude_scale)
            ml_adj["magnitude_dampened"] = True
            _cap_note["Price Model (suppl.)"] = f"magnitude ×{magnitude_scale:.2f} (out-of-sample direction accuracy {ml_dir_acc_b*100:.0f}% < 55%)"

    fv_ret_raw = float(fund_src["expected_return"]) if fund_src else None
    ml_ret_raw = float(ml_adj.get("expected_return", 0)) if ml_adj else None

    disagreement_flag = None
    disagreement_penalty = 1.0
    disagreement_gap = 0.0
    # A disagreement is evidence against the fundamental view only when the price model is itself
    # reliable. Below 55% out-of-sample direction accuracy its magnitude is already dampened above;
    # letting its disagreement also cut the fundamental source's confidence would penalize the
    # composite a second time for the price model's own weakness (adjustment BLEND_DISAGREEMENT).
    _pm_acc = float(ml_adj.get("backtest_dir_accuracy", 0.5) or 0.5) if ml_adj else None
    _pm_reliable = _pm_acc is not None and _pm_acc >= 0.55
    disagreement_gated = None

    if fv_ret_raw is not None and ml_ret_raw is not None:
        disagreement_gap = abs(fv_ret_raw - ml_ret_raw)
        same_sign = np.sign(fv_ret_raw) == np.sign(ml_ret_raw)
        _kind = None
        if disagreement_gap > 0.60 or (not same_sign and disagreement_gap > 0.30):
            _kind, _src_mult, _pen = "EXTREME_DISAGREEMENT", 0.40, 0.55
        elif disagreement_gap > 0.30 and same_sign:
            _kind, _src_mult, _pen = "STRONG_DISAGREEMENT", 0.65, 0.75
        elif not same_sign and disagreement_gap > 0.15:
            _kind, _src_mult, _pen = "DIRECTIONAL_CONFLICT", 0.70, 0.80
        if _kind:
            disagreement_flag = (f"{_kind} (FV:{fv_ret_raw*100:+.0f}% vs ML:{ml_ret_raw*100:+.0f}% "
                                 f"gap:{disagreement_gap*100:.0f}pp)")
            if ml_adj:
                ml_adj["confidence"] = ml_adj["confidence"] * _src_mult
                ml_adj["disagreement_flag"] = disagreement_flag
            if _pm_reliable:
                if fund_src:
                    fund_src["confidence"] = fund_src["confidence"] * _src_mult
                    fund_src["disagreement_flag"] = disagreement_flag
                disagreement_penalty = _pen
            else:
                disagreement_gated = (f"price model direction accuracy {(_pm_acc or 0)*100:.0f}% < 55%: its disagreement "
                                      "does not reduce the fundamental source's confidence")

    # LLM panel override: sharply downweight ML when panel agrees it is unreliable
    if llm and ml_adj:
        panel_unreliable = llm.get("override_ml", False)
        panel_unanimous = llm.get("vote", {}).get("unanimous", False)
        panel_ret = llm.get("expected_return", 0)
        ml_ret = ml_adj.get("expected_return", 0)
        disagreement = abs(ml_ret - panel_ret)

        if panel_unreliable and panel_unanimous and disagreement > 0.50:
            ml_adj["confidence"] = ml_adj["confidence"] * 0.15
            ml_adj["override_tier"] = "HARD_OVERRIDE"
        elif panel_unreliable and panel_unanimous:
            ml_adj["confidence"] = ml_adj["confidence"] * 0.30
            ml_adj["override_tier"] = "STRONG_OVERRIDE"
        elif panel_unreliable:
            ml_adj["confidence"] = ml_adj["confidence"] * 0.50
            ml_adj["override_tier"] = "MODERATE_OVERRIDE"
        elif disagreement > 0.40:
            ml_adj["confidence"] = ml_adj["confidence"] * 0.70
            ml_adj["override_tier"] = "SOFT_OVERRIDE"

    # Baseline weights (2026 rebalance): at 1yr horizon ~50–70% of return variance is
    # ΔEPS, which the FV path models directly. Direct-price ML retains a meaningful
    # share because it implicitly captures multiple-change dynamics that FV cannot.
    # Net effect: FV is now the primary signal, ML is the secondary "multiple-drift"
    # signal, macro is a cross-check, LLM is advisory.
    if llm_adj:
        fv_bw, macro_bw, ml_bw, llm_bw = 0.45, 0.10, 0.25, 0.20
    else:
        fv_bw, macro_bw, ml_bw, llm_bw = 0.50, 0.12, 0.38, 0.0

    _sign_conflict = False
    # the fair-value / price-model gap rules only apply when the price model is allowed to carry
    # weight: an excluded (UNRELIABLE / UNAVAILABLE, cap 0) model must not reshuffle the other
    # sources' weights or halve the headline through a "conflict" with its own output
    if fv_ret_raw is not None and ml_ret_raw is not None and (_qr_cap is None or _qr_cap > 0):
        gap = abs(fv_ret_raw - ml_ret_raw)
        same_sign = np.sign(fv_ret_raw) == np.sign(ml_ret_raw)

        if not same_sign and gap > 0.30:
            # Directional conflict: de-emphasise both FV and ML; lean on macro + LLM for tiebreak
            _sign_conflict = True
            if llm_adj:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.28, 0.15, 0.20, 0.37
            else:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.40, 0.25, 0.35, 0.0
        elif gap > 0.40:
            # Same-sign but large gap: trust FV's grounding in earnings, but keep ML weight
            # since multiple-change is more uncertain at extremes — split nearly evenly.
            if llm_adj:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.35, 0.10, 0.30, 0.25
            else:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.42, 0.12, 0.46, 0.0
            _sign_conflict = False
        elif gap > 0.25:
            # Moderate gap: hold FV emphasis, slight ML discount
            if llm_adj:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.40, 0.12, 0.28, 0.20
            else:
                fv_bw, macro_bw, ml_bw, llm_bw = 0.48, 0.14, 0.38, 0.0
            _sign_conflict = False
        else:
            _sign_conflict = False

    # Scanner lifecycle DCF (validated cross-sectional stack) — a peer valuation view to the deep FV.
    # Anchored on the walk-forward-validated EPS-ML forward EPS + lifecycle Damodaran assumptions;
    # its role is the same as in the scanner: a cross-check that tempers per-name FV optimism.
    scanner_flag = None
    sc_src = None
    if scanner and scanner.get("expected_return") is not None:
        sc_src = {"expected_return": float(np.clip(float(scanner["expected_return"]), -0.85, 1.50)),
                  "confidence": float(scanner.get("confidence", 0.30))}
        if fund_src:
            _fvr = float(fund_src["expected_return"])
            _gap = abs(sc_src["expected_return"] - _fvr)
            if np.sign(sc_src["expected_return"]) != np.sign(_fvr) and _gap > 0.30:
                scanner_flag = (f"VALUATION DIVERGENCE (lifecycle DCF {sc_src['expected_return']*100:+.0f}% "
                                f"vs multi-method FV {_fvr*100:+.0f}%, gap {_gap*100:.0f}pp)")
                sc_src["confidence"] *= 0.80
                fund_src["confidence"] = float(fund_src["confidence"]) * 0.80

    sources = {}
    # BlendSource kinds are declared where each source is created: what the view IS, and the objects it is built from.
    # "Multi-Method Fair Value" is the market-pricing composite's implied return (multiples + risk-adjusted analyst
    # target): market composite / price − 1. Intrinsic value enters only through the "Lifecycle DCF" source.
    source_kinds = {"Multi-Method Fair Value": ("MARKET_PRICING", ["pricing.market_composite", "market.price"]),
                    "Lifecycle DCF": ("INTRINSIC_VALUE", ["valuation.intrinsic_value", "market.price"]),
                    "Macro": ("MACRO", ["macro"]), "Price Model (suppl.)": ("PRICE_MODEL", ["quant.price_model"]),
                    "Analyst Panel": ("ANALYST_PANEL", ["scenarios.pm_return"])}
    if fund_src:
        sources["Multi-Method Fair Value"] = (fund_src["expected_return"], fund_src["confidence"], fv_bw)
    if sc_src:
        sources["Lifecycle DCF"] = (sc_src["expected_return"], sc_src["confidence"], 0.40)
    if macro_adj:
        sources["Macro"] = (macro_adj["expected_return"], macro_adj["confidence"], macro_bw)
    if ml_adj:
        # the reliability cap is applied LAST, after every disagreement/sign-conflict rebalance, so no
        # later weight reset can hand an unreliable model a large share
        if _qr_cap is not None and _qr_cap < 1.0:
            ml_bw = ml_bw * _qr_cap
            _cap_note["Price Model (suppl.)"] = ((_cap_note.get("Price Model (suppl.)") + "; ") if _cap_note.get("Price Model (suppl.)") else "") + \
                f"base weight ×{_qr_cap:.2f}: quant reliability {ml_adj['_reliability'].get('state')} ({ml_adj['_reliability'].get('basis')})"
        sources["Price Model (suppl.)"] = (ml_adj["expected_return"], ml_adj["confidence"], ml_bw)
    if llm_adj and llm_adj.get("expected_return") is not None:
        lc = float(llm_adj.get("confidence") or 0.1)
        # The panel reads the composite, the story value, the DCF, the EPS/revenue forecasts and the
        # peer table — it is interpretation of the same evidence, not independent evidence. Its
        # weight is discounted so shared inputs are not counted twice.
        _PANEL_CORR = 0.6
        sources["Analyst Panel"] = (float(llm_adj["expected_return"]), lc, llm_bw * _PANEL_CORR)

    if not sources:
        return None

    returns_list = [r for r, c, w in sources.values()]
    if len(returns_list) >= 3:
        med = np.median(returns_list)
        std_r = np.std(returns_list)
        if std_r > 0:
            dampened = {}
            for name, (ret, conf, base_w) in sources.items():
                z = abs(ret - med) / std_r
                if z > 3:
                    conf = conf * 0.2
                elif z > 2:
                    conf = conf * 0.5
                dampened[name] = (ret, conf, base_w)
            sources = dampened

    # Hard cap: macro effective weight ≤ 25% of total
    if "Macro" in sources:
        macro_ret, macro_c, macro_bw = sources["Macro"]
        macro_eff = macro_bw * macro_c
        other_eff = sum(bw * c for name, (r, c, bw) in sources.items() if name != "Macro")
        total_eff = macro_eff + other_eff
        if total_eff > 0 and macro_eff / total_eff > 0.25:
            target_macro_eff = 0.25 * other_eff / 0.75
            target_conf = target_macro_eff / macro_bw if macro_bw > 0 else 0
            sources["Macro"] = (macro_ret, min(macro_c, target_conf), macro_bw)

    total_weight = 0.0
    blended = 0.0
    for ret, conf, base_w in sources.values():
        w = base_w * conf
        blended += ret * w
        total_weight += w

    if total_weight > 0:
        blended /= total_weight
    _weighted_sum = float(blended)

    _adjustments = []
    if _sign_conflict:
        blended *= 0.50
        _adjustments.append({"name": "sign-conflict haircut", "factor": 0.5,
                             "note": (f"the market composite ({(fv_ret_raw or 0)*100:+.0f}%) and the price model ({(ml_ret_raw or 0)*100:+.0f}%) "
                                      f"disagree in sign by more than 30pp, so the weighted sum is halved")})

    returns = [r for r, c, w in sources.values()]
    std_ret = np.std(returns) if len(returns) > 1 else abs(blended) * 0.5
    low = max(-0.95, blended - 2.0 * std_ret)
    high = blended + 2.0 * std_ret
    avg_conf = float(np.mean([c for _, c, _ in sources.values()]))
    if fund_src and ml_adj:
        fv_rg = float(fund_src["expected_return"])
        ml_rg = float(ml_adj.get("expected_return", 0))
        gapg = abs(fv_rg - ml_rg)
        if gapg > 0.50:
            avg_conf *= 0.50
        elif gapg > 0.30:
            avg_conf *= 0.70
        elif gapg > 0.15:
            avg_conf *= 0.85
    if std_ret > 0.15:
        avg_conf *= 0.7

    if disagreement_penalty < 1.0:
        avg_conf *= disagreement_penalty

    if _sign_conflict:
        avg_conf *= 0.50

    mom_penalty = 1.0
    mom_flag = None
    if ml and ml.get("trailing_12m") is not None:
        t12m = ml["trailing_12m"]
        if t12m > 2.00:
            mom_penalty *= 0.50
            mom_flag = f"CYCLE_PEAK_RISK (12m:+{t12m*100:.0f}%)"
        elif t12m > 1.00:
            mom_penalty *= 0.70
            mom_flag = f"ELEVATED_MOMENTUM (12m:+{t12m*100:.0f}%)"
        elif t12m > 0.50:
            mom_penalty *= 0.85
            mom_flag = f"STRONG_MOMENTUM (12m:+{t12m*100:.0f}%)"
        elif t12m < -0.50:
            mom_penalty *= 0.75
            mom_flag = f"DEEP_DRAWDOWN (12m:{t12m*100:+.0f}%)"

    avg_conf = avg_conf * mom_penalty

    ml_dir_acc = float(ml.get("backtest_dir_accuracy", 0.5) or 0.5) if ml else 0.5
    conf_floor = 0.03 if ml_dir_acc < 0.40 else 0.10
    avg_conf = max(avg_conf, conf_floor)

    blended = float(np.clip(blended, -0.85, 3.00))
    low = max(-0.95, blended - 2.0 * std_ret)
    high = blended + 2.0 * std_ret

    if (
        disagreement_gap > 0.30
        and fv_ret_raw is not None
        and ml_ret_raw is not None
    ):
        range_min = min(fv_ret_raw, ml_ret_raw, blended)
        range_max = max(fv_ret_raw, ml_ret_raw, blended)
        low = min(low, range_min * 0.8)
        high = max(high, range_max * 1.1)

    low = max(-0.95, low)

    quant_conf_raw = float(np.clip(avg_conf, 0.0, 0.95))
    avg_conf = max(quant_conf_raw, 0.05)

    llm_fallback = False
    llm_fallback_return = None
    llm_fallback_conf = None
    if quant_conf_raw < 0.05 and llm_adj:
        llm_conf = float(llm_adj.get("confidence") or 0)
        if llm_conf > 0.30:
            llm_fallback = True
            llm_fallback_return = float(llm_adj.get("expected_return", 0))
            llm_fallback_conf = llm_conf

    direction = "BULLISH" if blended > 0.05 else "BEARISH" if blended < -0.05 else "NEUTRAL"
    return {
        "expected_return": blended,
        "low_estimate": low,
        "high_estimate": high,
        "confidence": avg_conf,
        "confidence_quant_raw": quant_conf_raw,
        "direction": direction,
        "sources": sources,
        "blend_note": "AI analyst panel weight ×0.6: the panel interprets the same inputs the market composite uses (multiples, targets, the intrinsic estimate, forecasts, peers), so it is not counted as fully independent evidence",
        "blend_audit": {
            "rows": [{"name": _n, "kind": source_kinds.get(_n, ("OTHER", []))[0],
                      "underlying_object_ids": source_kinds.get(_n, ("OTHER", []))[1], "return": float(_r), "raw_return": (float(_raw_ret[_n]) if _raw_ret.get(_n) is not None else float(_r)),
                      "cap_note": _cap_note.get(_n), "confidence": float(_c), "confidence_raw": (float(_raw_conf[_n]) if _raw_conf.get(_n) is not None else None),
                      "base_weight": float(_bw), "corr_penalty": (0.6 if _n == "Analyst Panel" else 1.0),
                      "eff_weight": float(_bw * _c), "share": float(_bw * _c) / (total_weight or 1.0),
                      "contribution": float(_r) * float(_bw * _c) / (total_weight or 1.0)}
                     for _n, (_r, _c, _bw) in sources.items()],
            "weighted_sum": _weighted_sum, "adjustments": _adjustments, "clip": [-0.85, 3.00], "headline": float(blended),
            "formula": "headline = clip( Σ view × share × adjustments ); share = base weight × confidence, normalised; the AI panel's base weight already carries its ×0.6 correlation penalty",
        },
        "std": std_ret,
        "momentum_confidence_flag": mom_flag,
        "momentum_confidence_penalty": mom_penalty,
        "disagreement_flag": disagreement_flag,
        "disagreement_gap": disagreement_gap,
        "disagreement_penalty": disagreement_penalty,
        "disagreement_gated": disagreement_gated,
        "scanner_flag": scanner_flag,
        "llm_fallback": llm_fallback,
        "llm_fallback_return": llm_fallback_return,
        "llm_fallback_conf": llm_fallback_conf,
    }


# ─── Display ────────────────────────────────────────────────────────────────




_JUDGE_SYSTEM_ROUND1_YEARLY = """You are an expert equity risk analyst evaluating a QUANTITATIVE 1-year trade signal for a human operator.

Your job is NOT to predict returns. You assess whether the quantitative signal is trustworthy and assign a CONFIDENCE ADJUSTMENT.

Key calibration for 1-YEAR horizon:
- At yearly horizon, FUNDAMENTALS matter most. Strong growth + reasonable valuation = credible. Sky-high P/E with decelerating growth = red flag.
- ML directional accuracy above 75% on 60 OOS samples for yearly prediction is SUSPICIOUS — academic state-of-the-art is ~55-60%. Flag inflated backtest metrics.
- Fair Value vs ML disagreement >50pp is a MAJOR red flag — one model must be wrong.
- MAE > 100% means the model's average error exceeds typical annual returns — treat as directional hint only.
- Insider selling is less relevant at yearly horizon (routine diversification), but 0 buys + heavy sells is still bearish.

Your confidence adjustment should be:
  +5 to +15: Signal looks clean, FV and ML broadly agree, fundamentals support thesis
  -5 to -15: Minor risks or moderate FV-ML gap (<40pp) — common for yearly predictions
  -15 to -25: Significant FV-ML gap (40-60pp) or walk-forward stability below 20%
  -25 to -40: Severe risks — FV-ML gap >60pp with directional conflict, or dir_acc below 35%
  Below -40: Critical — model appears broken (e.g., coin-flip accuracy with extreme prediction)

Important context for yearly (12-month) predictions:
- Walk-forward stability of 15-25% is NORMAL with 5 binary windows (not "extreme instability")
- FV-ML gaps of 30-50pp are COMMON for 12-month horizons — FLAG, don't REJECT
- 100% walk-forward dir accuracy on 5 windows = 5/5 correct — unusual but not overfitting proof
- "OOS:N(eff~K)" reports raw AND effective independent samples — judge trust by K (the eff~ value), not N. K<6 with high dir_acc is likely noise.
- REJECT only for: data quality issues, mathematical impossibilities, dir_acc <35% on 60+ samples, or >100pp FV-ML gap

*** EARNINGS-CYCLE RISK (critical for cyclicals — energy, materials, fertilizer, mining, semis) ***
- If the EARNINGS-CYCLE CONTEXT block shows current EPS well ABOVE its 10yr-normalized level (deviation > +50%), the fair value likely rests on cyclically-elevated (possibly peak) earnings. Converging FV and ML signals can BOTH be over-anchored on the same inflated base — agreement is NOT confirmation here.
- In that case, treat FV as an optimistic ceiling, apply a cyclical haircut (lean toward FLAG and a -15 to -25 adjustment even when FV-ML agree), and say so in your reasoning.
- The fundamental model may already have normalized EPS (shown in the context) — judge whether that normalization looks SUFFICIENT given how far above mid-cycle the company is.

*** EVENT / HEADLINE RISK ***
- If RECENT NEWS HEADLINES surface litigation, regulatory probes (DOJ/SEC/FTC), accounting concerns, guidance cuts, geopolitical/commodity-price shocks, or major capital-project execution risk, FLAG it explicitly — the quantitative signals are blind to these.
- A strong quant signal with a serious unaddressed headline risk should not be a clean PASS.

Respond with valid JSON only. No markdown."""

_JUDGE_USER_ROUND1_YEARLY = """CANDIDATE PROFILE (1-year horizon):
{ticker} @ ${current_price:.2f}
Blended expected return: {blend_ret_pct:+.2f}%
Walk-forward ML return: {ml_ret_pct:+.2f}%
Fair value estimate: ${fair_value:.2f} ({fv_ret_pct:+.2f}%)
FV-ML gap: {fv_ml_gap:.0f}pp
Direction: {direction} | Confidence: {confidence:.0f}%
Risk score (0-1, higher=worse): {risk_score:.2f}
Pick confidence (walk-forward): {pick_confidence:.1%}
ML dir accuracy: {dir_accuracy:.1%} (on {oos_samples} OOS samples from train_ml_model backtest)
Signal stability: {signal_stability:.1%} (fraction of 2-3 scan replay windows agreeing on direction — NOT accuracy on {oos_samples} samples)
MAE: {mae:.1f}%
P(return>10% in 252d): {p10:.1%} | P(return>0%): {ppos:.1%}
Earnings: {earn_str}
History: {history_days} days of price data
Key technicals: RSI {rsi:.0f} | BB%B {bb:.2f} | Mom12m {mom12m:+.2f}%

{extra_context}

Respond with ONLY this JSON:
{{
  "confidence_adjustment_pct": <int: -50 to +15>,
  "verdict": "<PASS|FLAG|REJECT>",
  "severity": <int: 1-5>,
  "risk_flags": ["<short flag>", "..."],
  "reasoning": "<1-2 sentences>",
  "confidence": <float 0-1>
}}"""

_JUDGE_SYSTEM_DEBATE_YEARLY = """You are the same risk analyst. Other models published their verdicts and confidence adjustments. You may revise your judgment.

Focus on: Did other analysts identify FV-ML contradictions you missed? Is the dir_acc plausibly high or likely overfitting? Are they correctly weighting fundamental quality vs. backtest metrics for a 1-YEAR horizon?

JSON only."""

_JUDGE_USER_DEBATE_YEARLY = """STOCK: {ticker}

YOUR ROUND-1 POSITION:
  Verdict: {my_verdict} | Adjustment: {my_adjustment:+d}% | Severity: {my_severity}/5
  Reasoning: {my_reasoning}

OTHER ANALYSTS:
{other_positions}

Respond with ONLY this JSON:
{{
  "confidence_adjustment_pct": <int: -50 to +15>,
  "verdict": "<PASS|FLAG|REJECT>",
  "severity": <int: 1-5>,
  "risk_flags": ["..."],
  "reasoning": "<1-2 sentences>",
  "confidence": <float 0-1>
}}"""




# ─── Scanner-aligned quant context (single name) ────────────────────────────


def scanner_quant_context(ticker, current_price=None, use_llm_dcf=True, verbose=False, fx_usd=1.0, market=None):
    """Scanner-aligned quant context for a single name — mirrors the validated damodaran_portfolio
    stack: style-balanced composite (value/quality/momentum sleeves, Sharpe 0.84 OOS) + universe
    rank, walk-forward-validated EPS-ML 12M-forward EPS (corr 0.41), deterministic trap guards,
    and the lifecycle LLM-Damodaran DCF anchored on the FORWARD EPS. Returns dict of the fields
    that could be computed; None if nothing could (ticker outside the Sharadar panel and SF1)."""
    try:
        import glob as _glob
        import damodaran_portfolio as dp
    except Exception as e:
        if verbose: print(f"    cross-sectional context unavailable: {str(e)[:60]}")
        return None
    tkr = ticker.upper(); ctx = {}
    # 1) style-balanced composite + universe rank (full cross-section, not just top-250)
    try:
        f = sorted(_glob.glob(f"{dp.PANEL}/*.parquet"))[-1]
        df = pd.read_parquet(f)
        keep = [c for c in ["week_ending","ticker","sector","marketcap"]+dp.STYLE_FACTORS if c in df.columns]
        cur = df[keep].copy(); cur["week_ending"] = pd.to_datetime(cur["week_ending"])
        cur = cur[cur.week_ending == cur.week_ending.max()].copy()
        sleeves = [dp._sleeve_z(cur, fl) for _, fl in dp.STYLE_SLEEVES]
        for (nm, _), sz in zip(dp.STYLE_SLEEVES, sleeves): cur["style_"+nm] = sz
        cur["composite"] = sum(sleeves)/len(sleeves)
        cur = cur[cur.marketcap > 0].sort_values("composite", ascending=False).reset_index(drop=True)
        cur["comp_rank"] = np.arange(1, len(cur)+1)
        row = cur[cur.ticker == tkr]
        if len(row):
            r = row.iloc[0]
            ctx.update(universe_n=int(len(cur)), comp_rank=int(r.comp_rank),
                       composite=float(r.composite), sector=str(r.sector),
                       styles={nm: float(r["style_"+nm]) for nm, _ in dp.STYLE_SLEEVES},
                       as_of=str(cur.week_ending.max().date()))
    except Exception as e:
        if verbose: print(f"    panel context failed: {str(e)[:60]}")
    # 2) EPS-ML forward EPS + trap guards + lifecycle valuation (SF1)
    try:
        m, sf1 = dp.train_eps_ml()
        feat = dp.latest_features(sf1, tkr)
        if feat is not None:
            X = pd.DataFrame([feat[dp.EPS_FEATS]]).replace([np.inf,-np.inf],np.nan).fillna(0.0).values
            g1 = float(m.predict(X)[0])
            eps0, basis, da_flag = dp.valuation_eps(feat, ctx.get("sector", "?"))  # FFO for REITs
            ctx["traps"] = dp.trap_flags(feat)
            ctx["val_basis"] = basis
            if da_flag: ctx["da_flag"] = da_flag
            if eps0 and eps0 == eps0 and eps0 > 0 and fx_usd and fx_usd != 1.0:
                eps0 = float(eps0) * float(fx_usd)      # Sharadar keeps foreign filers in reporting currency
            if eps0 and eps0 == eps0 and eps0 > 0 and current_price and current_price > 0:
                _pe0 = current_price / eps0
                if not (1.0 <= _pe0 <= 500.0):
                    # currency/unit break survived — a $100 stock does not earn $3,000/share.
                    if verbose: print(f"    12M earnings model skipped — implied P/E {_pe0:.2f} (currency/unit mismatch)")
                    eps0 = None
            if eps0 and eps0 == eps0 and eps0 > 0:
                ctx.update(eps_ttm=float(eps0), eps_fwd=float(eps0*(1+g1)), eps_growth=g1)
                price = current_price
                if (not price or price <= 0):
                    sh = feat.get("sharesbas") or feat.get("shareswadil")
                    mc = feat.get("marketcap")
                    if sh and sh == sh and sh > 0 and mc and mc == mc:
                        price = float(mc)/float(sh)
                if price and price > 0:
                    sector = ctx.get("sector", "?")
                    _mk = market or {}
                    _ke_mkt = ((_mk["rf"] + _mk["beta"] * _mk["erp"]) if (_mk.get("rf") is not None and _mk.get("beta")
                                                                           and _mk.get("erp")) else None)
                    pv, up, ke = dp.damodaran_value(eps0, g1, feat.get("roe"),
                                                    feat.get("payoutratio"), sector, price, ke=_ke_mkt)
                    if up == up:
                        ctx["dcf_fixed"] = dict(intrinsic=float(pv),
                                                upside=float(np.clip(up,-0.9,2.0)), ke=float(ke))
                    if use_llm_dcf:
                        rec = dict(ticker=tkr, sector=sector, price=float(price), pe0=float(price/eps0),
                                   eps0=float(eps0), eps_fwd=ctx["eps_fwd"], eps_growth=g1,
                                   roe_f=feat.get("roe"), net_marg=feat.get("net_marg"),
                                   op_marg=feat.get("op_marg"), rev_g1=feat.get("rev_g1"),
                                   de=feat.get("de"))
                        A = dp.llm_damodaran_assumptions(rec, verbose=verbose)
                        if A:
                            pv2, up2, ke2, stage = dp.dcf_from_assumptions(ctx["eps_fwd"], price, A, ke=_ke_mkt, rf=_mk.get("rf"))
                            if up2 == up2:
                                ctx["dcf_llm"] = dict(intrinsic=float(pv2), upside=float(up2),
                                                      ke=float(ke2), lifecycle=str(stage or "—"),
                                                      assumptions=A)
    except Exception as e:
        if verbose: print(f"    earnings-model/valuation context failed: {str(e)[:60]}")
    return ctx or None


# ─── Main ────────────────────────────────────────────────────────────────────


_CORP_EVENTS_SYSTEM = """You recall material corporate events from your knowledge. List events for the
given company from the LAST ~15 MONTHS that a stock analyst must not miss: CEO/CFO/chair transitions
(announced or effective), major M&A, spin-offs, stock splits, activist stakes, major litigation or
regulatory actions, large buyback/dividend policy changes.

Rules: ONLY include events you are HIGHLY confident actually happened — for smaller or obscure
companies return an empty list rather than guessing. Never invent dates. Max 5 events.
STRICT JSON only: {"events": [{"date": "YYYY-MM", "event": "<one plain-English line, <=25 words>"}]}"""


def _filed_segment_brief(all_data):
    """Segment economics FROM THE FILING (latest 10-Q/10-K XBRL), with each segment's industry
    benchmarks from Damodaran's datasets. Replaces the model-recalled brief whenever the company files
    two or more business-line segments."""
    # only segments that reconcile to the company, and only those of the valuation perimeter, reach the storytellers
    mix = all_data.get("segments_forward") or all_data.get("segments_canonical") or {}
    if mix.get("status") != "RECONCILED":
        return None
    segs = list(mix.get("segments") or [])
    if len(segs) < 2:
        return None
    import damodaran_data as dd
    bm_rows = (all_data.get("damodaran_inputs") or {}).get("business_mix") or []
    bm = {}
    for r in bm_rows:                    # one segment may be allocated to several industries: keep the largest
        if r.get("segment_id") not in bm or (r.get("fraction") or 1) > (bm[r["segment_id"]].get("fraction") or 1):
            bm[r.get("segment_id")] = r
    try:
        _months = max(1, round((datetime.fromisoformat(str(mix.get("period_end"))[:10]) -
                                datetime.fromisoformat(str(mix.get("period_start"))[:10])).days / 30.44))
    except (TypeError, ValueError):
        _months = None
    ev = dd.segment_build_evidence(segs, bm_rows)
    required, why = ev["required"], ev["reason"]
    rows = []
    for g in segs:
        b = bm.get(g.get("segment_id")) or {}
        rows.append({"segment_id": g.get("segment_id"), "name": g["name"], "revenue_share": round(g.get("revenue_share") or 0.0, 3),
                     "revenue_annualized_usd_b": (round(g["revenue"] * 12.0 / _months / 1e9, 2) if (_months and g.get("revenue")) else None),
                     "margin": (round(g["margin"], 3) if g.get("margin") is not None else None), "margin_basis": "operating",
                     "growth": g.get("growth"), "capex_to_revenue": g.get("capex_to_revenue"),
                     "product_lines": [p["name"] for p in g.get("product_lines") or []],
                     "industry": b.get("industry"), "industry_margin": b.get("industry_operating_margin"),
                     "industry_sales_to_capital": b.get("industry_sales_to_capital")})
    return {"as_of": mix.get("period_end"), "period": f"{mix.get('period_start')} to {mix.get('period_end')}",
            "source": mix.get("source"), "filed": True, "segments": rows, "absorbed": [],
            "segment_build_required": required, "required_reason": why, "required_evidence": ev}


def _gaap_reblend(fair_val_result, _stage_check, current_price):
    """GAAP-distortion re-blend of the market composite: reported EPS far from adjusted EPS means the statement-earnings
    multiples (P/E, PEG) misstate the business, so their confidence is cut and the composite re-blended, recorded as a
    composite step. It runs AFTER the sanity cap, so it is bounded by the same recorded cap (a later step may never
    leave the composite outside the bound the audit records)."""
    _meths = fair_val_result["methods"]
    _hit = [m for m in ("P/E", "PEG") if m in _meths]
    for m in _hit:
        _meths[m]["conf"] = (_meths[m].get("conf") or 0.3) * 0.35
    _tw = sum((v.get("conf") or 0.0) for v in _meths.values())
    if _hit and _tw > 0:
        _newfp = sum((v.get("price") or 0.0) * (v.get("conf") or 0.0) for v in _meths.values()) / _tw
        _ca = fair_val_result.get("composite_audit")
        # the re-blend comes AFTER the sanity cap, so it is bounded by the same recorded cap (PGY: the cap
        # set 2.5× the price, then the re-blend lifted the composite to 3.0× and the state failed S6)
        _lo = float((_ca or {}).get("cap_lower") or 0.2); _hi = float((_ca or {}).get("cap_upper") or 5.0)
        _blend_fp = _newfp
        _newfp = max(_lo * current_price, min(_hi * current_price, _newfp))
        fair_val_result["fair_price"] = _newfp
        fair_val_result["expected_return"] = (_newfp - current_price) / current_price
        if isinstance(_ca, dict):
            _ca.setdefault("steps", []).append({"step": "GAAP-distortion re-blend", "set_to": float(_blend_fp),
                "note": (f"{_stage_check['distortion_reason']}; {', '.join(_hit)} confidence ×0.35 and the composite "
                         "re-blended as Σ price × confidence / Σ confidence over all methods: " +
                         ", ".join(f"{k} {v.get('price', 0):,.0f}×{(v.get('conf') or 0):.2f}" for k, v in _meths.items()))})
            if _newfp != _blend_fp:
                _ca["steps"].append({"step": "sanity cap", "set_to": float(_newfp),
                                     "note": (f"after the GAAP-distortion re-blend: composite limited to {_hi:.2f}x / "
                                              f"{_lo:.2f}x of the price for this market-cap and volatility tier "
                                              f"(re-blend {_blend_fp / current_price:.2f}x)")})
            for _row in _ca.get("methods") or []:
                _mv = _meths.get(_row["method"]) or {}
                _row["share_final"] = float((_mv.get("conf") or 0.0) / _tw)
                _row["in_composite"] = True
            _ca["final"] = float(_newfp); _ca["final_vs_price"] = float(_newfp / current_price - 1.0)
        print(f"  [gaap] {_stage_check['distortion_reason']} — {', '.join(_hit)} down-weighted; composite re-blended to ${_newfp:.2f}")


def _business_decomposition(ticker, all_data):
    """BusinessDecompositionState (business.py) from the canonical segment state, the canonical corporate-action state
    and the story base: the ONLY source of business identity for the valuation story. Model recall is never one (APP:
    a recalled segment mix put a business sold the year before into the story)."""
    import business as _bz, corporate_actions as _cam
    cas = _cam.build(ticker, all_data.get("restatement"), all_data.get("edgar_actions"), all_data.get("corp_actions"),
                     all_data.get("segments_canonical"))
    return _bz.build(ticker, all_data.get("segments_canonical"), cas, all_data.get("story_base"),
                     (all_data.get("perimeter_adjustment") or {}).get("segment_ids"))


def _product_line_brief(decomp):
    """The storytellers' brief for a finer decomposition the company itself files (the product/service lines of its one
    reportable segment): names, ids and revenue shares from the filing. None for any other decomposition."""
    import business as _bz
    d = decomp or {}
    if d.get("source_type") != _bz.FILED_PRODUCT_LINES or d.get("mode") != _bz.MULTI_BUSINESS:
        return None
    el = _bz.eligible(d)
    tot = sum((b.get("revenue") or 0) for b in el)
    if tot <= 0:
        return None
    return {"filed": True, "source": d.get("source"), "period": f"filed {d.get('source_date')}", "as_of": d.get("source_date"),
            "segments": [{"segment_id": b["business_id"], "name": b["name"], "revenue_share": round((b.get("revenue") or 0) / tot, 3),
                          "margin": None, "margin_basis": None, "growth": None, "capex_to_revenue": None, "product_lines": []}
                         for b in el],
            "absorbed": [], "segment_build_required": False, "required_reason": "filed product lines of one reportable segment",
            "required_evidence": {"required": False}}


_CORP_ACTIONS_SYSTEM = """You recall STRUCTURAL corporate actions for the given company from your knowledge:
spin-offs, divestitures, and large acquisitions (>5% of revenue) COMPLETED in the last 18 months or
currently PENDING. These change what the reported financials describe, so an analyst must know them.

Rules: ONLY include actions you are HIGHLY confident about; for smaller or obscure companies return
an empty list rather than guessing. Give the EFFECTIVE (closing) date, not the announcement date, when
known. approx_revenue_share = the unit's share of the company's revenue before the action (0-1), or
null if unsure. Max 4 actions.
STRICT JSON only:
{"actions": [{"kind": "spin_off|divestiture|acquisition", "business": "<unit or target>",
              "status": "completed|pending", "effective": "YYYY-MM-DD or YYYY-MM or null",
              "approx_revenue_share": <0-1 or null>, "note": "<=15 words"}]}"""


def _corporate_actions_brief(ticker, name, call_json, headlines=None):
    """Structural corporate actions (spin-offs / divestitures / big acquisitions) with effective
    dates. The events brief only recalled S&P Global's 2025 spin-off ANNOUNCEMENT; the spin
    completed on 2026-07-01, the statements still contained the unit, and the story valued the
    divested business as a segment. Returns a list of normalized action dicts or []."""
    try:
        _hl = ("\nRecent headlines (data — ADD any acquisition/divestiture/spin-off they announce, status 'pending' "
               "unless a headline says it closed):\n" + "\n".join(f"- {h}" for h in headlines if h)) if headlines else ""
        out = call_json(_CORP_ACTIONS_SYSTEM, f"Company: {name or ticker} (ticker {ticker}). Today: {datetime.now().strftime('%Y-%m-%d')}.{_hl}")
        acts = []
        for a in ((out or {}).get("actions") or [])[:4]:
            if not isinstance(a, dict) or not a.get("business"):
                continue
            kind = str(a.get("kind") or "").lower()
            if kind not in ("spin_off", "divestiture", "acquisition"):
                continue
            sh = a.get("approx_revenue_share")
            try:
                sh = float(sh) if sh is not None else None
                if sh is not None and sh > 1.0: sh /= 100.0
                if sh is not None and not (0.0 < sh < 0.9): sh = None
            except (TypeError, ValueError):
                sh = None
            acts.append({"kind": kind, "business": str(a["business"])[:60],
                         "status": ("completed" if str(a.get("status") or "").lower().startswith("comp") else "pending"),
                         "effective": (str(a.get("effective"))[:10] if a.get("effective") else None),
                         "revenue_share": sh, "note": str(a.get("note") or "")[:100]})
        return acts
    except Exception:  # noqa: BLE001
        return []


def _restatement_from_actions(actions, statements_through):
    """A completed spin-off/divestiture whose effective date falls inside (or after) the trailing
    twelve months covered by the statements means the reported TTM revenue and margins still
    include the divested business. Returns the caveat dict or None."""
    try:
        if not actions or not statements_through:
            return None
        _end = datetime.strptime(str(statements_through)[:10], "%Y-%m-%d")
        _start = _end - timedelta(days=365)
        for a in actions:
            if a.get("kind") not in ("spin_off", "divestiture") or a.get("status") != "completed":
                continue
            eff = a.get("effective")
            if not eff:
                continue
            try:
                _eff = datetime.strptime(eff[:10] if len(eff) >= 10 else eff[:7] + "-01", "%Y-%m-%d")
            except ValueError:
                continue
            if _eff > _start:                       # divested inside the TTM window or after the last statement
                after = _eff > _end
                return {"kind": a["kind"], "business": a["business"], "effective": eff,
                        "revenue_share": a.get("revenue_share"), "statements_through": str(statements_through)[:10],
                        "note": (f"{a['business']} was {'spun off' if a['kind'] == 'spin_off' else 'divested'} effective {eff}"
                                 + (f", after the latest reported quarter ({str(statements_through)[:10]})" if after
                                    else f", inside the trailing-twelve-month window ending {str(statements_through)[:10]}")
                                 + "; reported TTM revenue, margins and multiples still include it"
                                 + (f" (~{a['revenue_share']*100:.0f}% of revenue)" if a.get("revenue_share") else "") + ".")}
        return None
    except Exception:  # noqa: BLE001
        return None


_EDGAR_UA = "Crucible equity research tool (contact: sameer.mohd@gmail.com)"
DAMODARAN_ERP = 0.0428               # implied equity risk premium (documented model constant, not a data substitute)


def _edgar_actions(ticker, months=18):
    """Canonical corporate-action evidence from SEC EDGAR: recent 8-K filings whose items include
    2.01 (Completion of Acquisition or Disposition of Assets) or 1.01 (Material Definitive
    Agreement), with a short excerpt of the Item 2.01 text. Model recall had S&P Global's
    July-2026 Mobility spin as "pending" three runs in a row; the 8-K filed 2026-07-02 settles it.
    Returns a list of {filed, items, kind, excerpt, url} (newest first) or []."""
    import urllib.request as _u, re as _re, json as _j
    out = []
    try:
        def _get(url, timeout=12):
            try:
                import requests as _rq          # carries its own CA bundle (certifi)
                r = _rq.get(url, headers={"User-Agent": _EDGAR_UA}, timeout=timeout)
                r.raise_for_status()
                return r.content
            except ImportError:
                req = _u.Request(url, headers={"User-Agent": _EDGAR_UA})
                with _u.urlopen(req, timeout=timeout) as r:
                    return r.read()
        # ticker → CIK (cached daily); the canonical layer usually loaded the submissions already
        cik = _EDGAR_CIK.get(ticker.upper())
        if cik and _EDGAR_SUB.get(cik):
            _ct = {}
        try:
            if cik and _EDGAR_SUB.get(cik):
                raise StopIteration
            _cf = CACHE_DIR / "edgar_company_tickers.json"
            if _cf.exists() and (datetime.now().timestamp() - _cf.stat().st_mtime) < 7 * 86400:
                _ct = _j.loads(_cf.read_text())
            else:
                _ct = _j.loads(_get("https://www.sec.gov/files/company_tickers.json"))
                try: _cf.write_text(_j.dumps(_ct))
                except Exception: pass
        except StopIteration:
            _ct = {}
        except Exception:
            _ct = _j.loads(_get("https://www.sec.gov/files/company_tickers.json"))
        for v in _ct.values():
            if str(v.get("ticker", "")).upper() == ticker.upper():
                cik = int(v["cik_str"]); break
        if cik is None:
            return []
        _EDGAR_CIK[ticker.upper()] = cik
        sub = _EDGAR_SUB.get(cik) or _j.loads(_get(f"https://data.sec.gov/submissions/CIK{cik:010d}.json"))
        _EDGAR_SUB[cik] = sub
        rec = sub.get("filings", {}).get("recent", {})
        forms, dates, items, accs, docs = (rec.get("form", []), rec.get("filingDate", []), rec.get("items", []),
                                           rec.get("accessionNumber", []), rec.get("primaryDocument", []))
        cutoff = (datetime.now() - timedelta(days=30 * months)).strftime("%Y-%m-%d")
        for f, dt, it, acc, doc in zip(forms, dates, items, accs, docs):
            if f != "8-K" or dt < cutoff:
                continue
            its = [x.strip() for x in str(it or "").split(",")]
            kind = ("completed_acquisition_or_disposition" if "2.01" in its
                    else ("material_agreement" if ("1.01" in its and "2.03" not in its) else None))
            if not kind:
                continue
            url = f"https://www.sec.gov/Archives/edgar/data/{cik}/{acc.replace('-', '')}/{doc}"
            excerpt = None
            if kind == "completed_acquisition_or_disposition":
                try:
                    html = _get(url).decode("utf-8", "ignore")
                    txt = _re.sub(r"<[^>]+>", " ", html); txt = _re.sub(r"&nbsp;|&#160;", " ", txt); txt = _re.sub(r"\s+", " ", txt)
                    import html as _html
                    txt = _html.unescape(txt)
                    # the Item 2.01 paragraph itself, or (when it is just a cross-reference) the
                    # first sentence in the filing that mentions completion/separation/closing
                    m = _re.search(r"Item\s*2\.01\.?\s*(?:Completion of Acquisition or Disposition of Assets\.?)?\s*(.{60,700}?)(?=\s*Item\s*\d\.\d\d\b|$)", txt, _re.I)
                    cand = m.group(1).strip() if m else ""
                    if len(cand) < 60 or cand.lower().startswith("item "):
                        m2 = _re.search(r"([^.]{0,200}\b(?:completed|completion of|consummated|closing of)\b[^.]{0,260}\.)", txt, _re.I)
                        cand = m2.group(1).strip() if m2 else cand
                    excerpt = cand[:420] or None
                except Exception:
                    excerpt = None
            out.append({"filed": dt, "items": its, "kind": kind, "excerpt": excerpt, "url": url})
            if len(out) >= 6:
                break
    except Exception:  # noqa: BLE001
        return out
    return out


_EDGAR_CIK = {}
_EDGAR_SUB = {}


def _edgar_officer_changes(ticker, months=30):
    """8-K Item 5.02 filings (officer/director changes) with the Item 5.02 paragraph and any
    'effective <date>' dates in it — the canonical source for a CEO transition date. Recall
    put Martina Cheung's start in November 2025; the 8-K of 2024-06-27 says November 1, 2024."""
    import re as _re, html as _html
    try:
        import requests as _rq
        cik = _EDGAR_CIK.get(ticker.upper()); sub = _EDGAR_SUB.get(cik) if cik else None
        if not sub:
            return []
        rec = sub.get("filings", {}).get("recent", {})
        cutoff = (datetime.now() - timedelta(days=30 * months)).strftime("%Y-%m-%d")
        out = []
        for f, dt, it, acc, doc in zip(rec.get("form", []), rec.get("filingDate", []), rec.get("items", []),
                                       rec.get("accessionNumber", []), rec.get("primaryDocument", [])):
            if f != "8-K" or dt < cutoff or "5.02" not in str(it or ""):
                continue
            url = f"https://www.sec.gov/Archives/edgar/data/{cik}/{acc.replace('-', '')}/{doc}"
            try:
                t = _rq.get(url, headers={"User-Agent": _EDGAR_UA}, timeout=15).text
            except Exception:
                continue
            t = _html.unescape(_re.sub(r"\s+", " ", _re.sub(r"<[^>]+>", " ", t)))
            m = _re.search(r"Item\s*5\.02\.?(.{0,900})", t, _re.I)
            ex = (m.group(1) if m else t[:600]).strip()
            ex = _re.sub(r"^Departure of Directors or Certain Officers;.*?Officers\.\s*", "", ex)
            dates = []
            for dm in _re.finditer(r"effective\s+(?:as\s+of\s+|on\s+)?(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s+(20\d\d)", ex, _re.I):
                try:
                    dates.append(datetime.strptime(f"{dm.group(1)} {dm.group(2)} {dm.group(3)}", "%B %d %Y").strftime("%Y-%m-%d"))
                except ValueError:
                    pass
            # a CEO TRANSITION, not any filing that merely mentions the CEO (a Chief Legal Officer's
            # retirement letter "reporting to the CEO" was being picked up)
            _ceo_tr = bool(_re.search(r"(retir\w*|appoint\w*|succeed\w*|named|elected|transition|step(?:ping)? down)[^.]{0,120}?(chief executive officer|\bCEO\b)"
                                      r"|(chief executive officer|\bCEO\b)[^.]{0,80}?(transition|retir\w*|appoint\w*|succeed\w*|named|elected)", ex, _re.I))
            out.append({"filed": dt, "excerpt": ex[:520], "effective_dates": dates, "url": url, "ceo_related": _ceo_tr})
            if len(out) >= 8:
                break
        return out
    except Exception:  # noqa: BLE001
        return []


def _edgar_form4(ticker, months=6, max_filings=15):
    """Insider transactions from the issuer's SEC Form 4 XMLs: transaction code (S open-market sale,
    P purchase, M option exercise, F tax withholding, A award, G gift), shares, price, holdings
    after, and the Rule 10b5-1 flag/footnote. The market-data provider's rows carry none of this,
    so a CEO's pre-scheduled plan sales read as discretionary 'insider selling' (JLL, Aug-2026)."""
    import re as _re, html as _html
    out = []
    try:
        import requests as _rq
        cik = _EDGAR_CIK.get(ticker.upper()); sub = _EDGAR_SUB.get(cik) if cik else None
        if not sub:
            return out
        rec = sub.get("filings", {}).get("recent", {})
        cutoff = (datetime.now() - timedelta(days=30 * months)).strftime("%Y-%m-%d")
        for f, dt, acc, doc in zip(rec.get("form", []), rec.get("filingDate", []), rec.get("accessionNumber", []), rec.get("primaryDocument", [])):
            if f not in ("4", "4/A") or dt < cutoff:
                continue
            base = f"https://www.sec.gov/Archives/edgar/data/{cik}/{acc.replace('-', '')}/"
            fn = doc.split("/")[-1] if doc else "form4.xml"
            if not fn.endswith(".xml"):
                fn = "form4.xml"
            try:
                x = _rq.get(base + fn, headers={"User-Agent": _EDGAR_UA}, timeout=15).text
            except Exception:
                continue
            if "<ownershipDocument" not in x:
                continue
            def _g(tag, blob=x):
                m = _re.search(rf"<{tag}>\s*(?:<value>)?([^<]+)", blob)
                return _html.unescape(m.group(1).strip()) if m else None
            owner = _g("rptOwnerName"); title = _g("officerTitle") or ("Director" if _g("isDirector") in ("1", "true") else "")
            if owner:                                  # EDGAR conformed name "Ulbrich Christian" → "Christian Ulbrich"
                if owner.isupper():
                    owner = " ".join(w.capitalize() for w in owner.split())
                if "," in owner:
                    _l, _f = [x.strip() for x in owner.split(",", 1)]; owner = f"{_f} {_l}"
                else:
                    _ps = owner.split()
                    if len(_ps) >= 2: owner = " ".join(_ps[1:] + _ps[:1])
            plan_flag = _g("aff10b5One") in ("1", "true")
            fns = {m.group(1): _re.sub(r"\s+", " ", _re.sub(r"<[^>]+>", " ", m.group(2))).strip() for m in _re.finditer(r'<footnote id="([^"]+)">(.*?)</footnote>', x, _re.S)}
            for m in _re.finditer(r"<nonDerivativeTransaction>(.*?)</nonDerivativeTransaction>", x, _re.S):
                b = m.group(1)
                fids = _re.findall(r'footnoteId id="([^"]+)"', b)
                ftxt = " ".join(fns.get(i, "") for i in fids)
                plan = plan_flag or bool(_re.search(r"10b5-1", ftxt))
                adopted = None
                am = _re.search(r"adopted\s+(?:by\s+[^,.]{0,40}?\s+)?on\s+([A-Z][a-z]+\s+\d{1,2},\s+20\d\d)", ftxt, _re.I)
                if am:
                    try: adopted = datetime.strptime(am.group(1), "%B %d, %Y").strftime("%Y-%m-%d")
                    except ValueError: adopted = am.group(1)
                sh = safe_float(_g("transactionShares", b)); px = safe_float(_g("transactionPricePerShare", b))
                out.append({"filed": dt, "date": _g("transactionDate", b), "owner": owner, "title": title,
                            "code": _g("transactionCode", b), "ad": _g("transactionAcquiredDisposedCode", b),
                            "shares": sh, "price": px, "value": (sh or 0) * (px or 0),
                            "post": safe_float(_g("sharesOwnedFollowingTransaction", b)), "plan": plan, "adopted": adopted,
                            "url": base + fn})
            if len({o["url"] for o in out}) >= max_filings:
                break
    except Exception:  # noqa: BLE001
        pass
    return out


def _insider_canon(form4, insider_rows=None, months=6):
    """One canonical insider object every persona consumes (transaction codes, plan sales, size).

    The structured classification and the type-weighted factor come from ``insider``: a scheduled
    Rule 10b5-1 sale, a tax-withholding disposition and a discretionary open-market sale are three
    different economic events and no longer collapse into one 'insider selling' number."""
    import insider as _ins
    tx = [t for t in (form4 or []) if t.get("code")]
    c = {"window_months": months, "n_transactions": len(tx), "source": "SEC Form 4 filings (issuer submissions)", "transactions": tx[:40]}
    c["classified"] = _ins.classify_all(tx)
    c["factor"] = _ins.insider_factor(tx)
    c["characterization"] = _ins.characterization(c["factor"])
    c["plan_only"] = _ins.plan_only(c["factor"])
    # the direction of discretionary trading only; the filing never records a motive, so none may be inferred
    c["discretionary_activity_signal"] = (
        "NONE" if c["plan_only"] or c["factor"].get("score") is None
        else ("NEGATIVE" if c["factor"]["score"] < 0 else "POSITIVE"))
    c["motive_inference_allowed"] = _ins.MOTIVE_INFERENCE_ALLOWED
    c["governance_signal"] = "NEUTRAL"
    if not tx:
        return c
    val = lambda code, plan=None: sum(t["value"] for t in tx if t["code"] == code and (plan is None or bool(t["plan"]) == plan))
    c["open_market_sale_value"] = val("S", False); c["plan_sale_value"] = val("S", True)
    c["purchase_value"] = val("P"); c["option_exercise_value"] = val("M"); c["tax_withholding_value"] = val("F")
    c["award_shares"] = sum((t["shares"] or 0) for t in tx if t["code"] == "A"); c["gift_shares"] = sum((t["shares"] or 0) for t in tx if t["code"] == "G")
    tot_s = c["open_market_sale_value"] + c["plan_sale_value"]
    c["plan_share"] = (c["plan_sale_value"] / tot_s) if tot_s > 0 else None
    # largest seller by value
    by = {}
    for t in tx:
        if t["code"] != "S": continue
        k = (t["owner"], t["title"])
        d = by.setdefault(k, {"name": t["owner"], "title": t["title"], "shares": 0.0, "value": 0.0, "plan_all": True, "adopted": None, "post": None, "dates": []})
        d["shares"] += t["shares"] or 0; d["value"] += t["value"] or 0; d["plan_all"] &= bool(t["plan"]); d["adopted"] = d["adopted"] or t["adopted"]
        d["post"] = t["post"] if t["post"] is not None else d["post"]; d["dates"].append(t["date"])
    if by:
        top = max(by.values(), key=lambda d: d["value"])
        held = (top["post"] or 0) + top["shares"]
        top["pct_of_holdings"] = (top["shares"] / held) if held > 0 else None
        top["first"], top["last"] = min(top["dates"]), max(top["dates"])
        c["top_seller"] = top
        who = f"{top['title'] or 'insider'} {top['name']}".strip()
        size = f"{top['shares']:,.0f} shares (~${top['value']/1e6:.1f}M" + (f", about {top['pct_of_holdings']*100:.0f}% of holdings)" if top.get("pct_of_holdings") else ")")
        when = top["first"] if top["first"] == top["last"] else f"{top['first']} to {top['last']}"
        if top["plan_all"]:
            c["note"] = (f"{who} sold {size} between {when} under a previously adopted Rule 10b5-1 trading plan" + (f" (adopted {top['adopted']})" if top.get("adopted") else "")
                         + (f"; open-market discretionary sales by insiders in the window: ${c['open_market_sale_value']/1e6:.1f}M" if c["open_market_sale_value"] > 0 else "; no discretionary open-market insider sales in the window")
                         + "; the transactions contribute to the model's negative insider-activity factor but should not be read as a discretionary valuation signal")
        else:
            c["note"] = (f"{who} sold {size} between {when} in open-market transactions (not flagged as Rule 10b5-1 plan sales)"
                         + (f"; a further ${c['plan_sale_value']/1e6:.1f}M of sales were under 10b5-1 plans" if c["plan_sale_value"] > 0 else ""))
    else:
        parts = []
        if c["purchase_value"]: parts.append(f"open-market purchases ${c['purchase_value']/1e6:.1f}M")
        if c["option_exercise_value"]: parts.append(f"option exercises ${c['option_exercise_value']/1e6:.1f}M")
        if c["tax_withholding_value"]: parts.append(f"tax-withholding dispositions ${c['tax_withholding_value']/1e6:.1f}M (not sales)")
        if c["award_shares"]: parts.append(f"equity awards {c['award_shares']:,.0f} shares")
        if c["gift_shares"]: parts.append(f"gifts {c['gift_shares']:,.0f} shares")
        c["note"] = ("no open-market insider sales in the window; " + ", ".join(parts)) if parts else "no reportable insider transactions in the window"
    return c


def _leadership_canon(edgar_officers, wiki_str, form4, issuer=None, officers_20f=None):
    """Canonical leadership under the issuer's own filing regime.

    Candidates are gathered from every available source and ``leadership.resolve`` ranks them for
    this regime: Form 4 titles and 8-K Item 5.02 lead for a US domestic filer, while a foreign
    private issuer — which files neither — is led by the 20-F management section and 6-K management
    changes. A source that reports an appointment EVENT can confirm a role but never establishes
    that a long-serving officer's seat is empty."""
    import re as _re
    import leadership as _ld
    cands = {}
    def _set(title, name, source, effective=None, rank=0, source_id="secondary_reference", filed=None, ttl=None):
        if not name: return
        cands.setdefault(title, []).append(
            {"name": name.strip(), "source_id": source_id, "effective": effective, "title": ttl,
             "filed": filed or (source.split("filed ")[-1] if "filed " in source else "")})
    tmap = [("CEO", r"chief\s+executive|\bCEO\b"), ("CFO", r"chief\s+financial|\bCFO\b"), ("COO", r"chief\s+operating|\bCOO\b")]
    # 1) Form 4 officer titles — the reporting person states their own current title
    for t in sorted(form4 or [], key=lambda t: t.get("filed") or "", reverse=True):
        title = t.get("title") or ""
        for key, pat in tmap:
            # firm-level roles only: "CEO CCB", "CEO Consumer & Community Banking", "CFO of the Investment
            # Bank" are business-unit roles and must never resolve to the company's CEO/CFO/COO
            _tnorm = _re.sub(r"[,.]", " ", title).strip()
            _unit_words = r"former|interim|deputy|co-?(?:ceo|cfo|coo|chief)|assistant|division|divisional|regional|americas|emea|apac|international|global|consumer|community|commercial|corporate|investment bank|banking|markets|asset|wealth|card|payments|treasury|technology|operations|ccb|cib|awm|cb\b|of\s+(?!the\s+company)"
            _firm_ok = (_re.fullmatch(r"(?:(?:co-?founder|founder|executive\s+chairman|vice\s+chairman|chairman|chair|president|director)\s*(?:and|&|,)?\s*)*(?:chief\s+(?:executive|financial|operating)\s+officer|ceo|cfo|coo)(?:\s*(?:and|&|,)\s*(?:president|chairman|chair|director|co-?founder|founder))*", _tnorm, _re.I) is not None)
            if _re.search(pat, title, _re.I) and _firm_ok and not _re.search(_unit_words, title, _re.I):
                _set(key, t.get("owner") or "", f"SEC Form 4 filed {t.get('filed')}", rank=3,
                     source_id="sec_form4_officer_title", filed=t.get("filed"), ttl=title)
    # 2) 8-K Item 5.02 appointments
    for o in sorted(edgar_officers or [], key=lambda o: o.get("filed") or ""):
        ex = o.get("excerpt") or ""
        for key, pat in tmap:
            m = _re.search(rf"(?:appoint\w*|named|elected|promot\w*)\s+(?:of\s+)?(?:Mr\.|Ms\.|Mrs\.|Dr\.)?\s*([A-Z][a-zA-Z.'’-]+(?:\s+[A-Z][a-zA-Z.'’-]+){{1,2}})\s*(?:,\s*(?:age\s+)?\d+,)?\s*(?:as|to\s+(?:serve\s+as|the\s+position\s+of|be)|to)\s+(?:the\s+Company's\s+|its\s+|the\s+)?(?:new\s+)?(?:{pat})", ex, _re.I)
            if not m:
                m = _re.search(rf"([A-Z][a-zA-Z.'’-]+(?:\s+[A-Z][a-zA-Z.'’-]+){{1,2}})\s+(?:will|would|has\s+been|was)\s+(?:appointed|named|elected|promoted|succeed\w*)[^.]{{0,40}}?(?:{pat})", ex, _re.I)
            if m:
                eff = (o.get("effective_dates") or [None])[-1]
                _set(key, m.group(1), f"SEC 8-K Item 5.02 filed {o.get('filed')}", effective=eff, rank=2,
                     source_id="sec_8k_item502", filed=o.get("filed"))
    # 3) company profile (Wikipedia infobox) fallback
    w = str(wiki_str or "")
    for key, pat in tmap:
        m = _re.search(rf"([A-Z][a-zA-Z.'’-]+(?:\s+[A-Z][a-zA-Z.'’-]+){{1,2}})\s*\(([^)]*(?:{pat})[^)]*)\)", w, _re.I)
        if m:
            _set(key, m.group(1), "company profile (Wikipedia infobox)", rank=1, source_id="secondary_reference")
    # 4) 20-F management section — the leading state source for a foreign private issuer
    for _t20, _o20 in (officers_20f or {}).items():
        if _t20 in ("CEO", "CFO", "COO") and isinstance(_o20, dict) and _o20.get("name"):
            _set(_t20, _o20["name"], _o20.get("source") or "SEC 20-F", rank=4,
                 source_id="sec_20f_management", filed=_o20.get("filed"), ttl=_o20.get("title"))
    canon = _ld.resolve(cands, issuer)
    canon["as_of"] = datetime.now().strftime("%Y-%m-%d")
    canon["_regime"] = _ld.regime(issuer)
    canon["_source_priority"] = [x["source_id"] for x in _ld.source_priority(issuer)]
    return canon


def _momentum_canon(current_price, tech, overview):
    """Mechanical momentum definition shared by the personas, the synthesis line and the gate:
    positive = price above the 200-day AND 50-day above the 200-day AND 3-month return ≥ 0;
    negative = below the 200-day AND 3-month return < 0; otherwise neutral/mixed."""
    try:
        s = (tech or {}).get("signals") or {}
        ov = overview or {}
        sma50 = safe_float(ov.get("50DayMovingAverage")); sma200 = safe_float(ov.get("200DayMovingAverage"))
        px = float(current_price)
        above200 = (px > sma200) if sma200 else (s.get("trend_sma200", 0) > 0)
        gc = (sma50 > sma200) if (sma50 and sma200) else (s.get("golden_cross", 0) > 0)
        m3 = s.get("momentum_3m"); m12 = s.get("momentum_12m")
        positive = bool(above200 and gc and (m3 is None or m3 >= 0))
        negative = bool((not above200) and (m3 is not None and m3 < 0))
        label = "positive" if positive else "negative" if negative else "neutral/mixed"
        bits = []
        if sma200: bits.append(f"price ${px:,.0f} {'above' if above200 else 'below'} the 200-day (${sma200:,.0f})")
        if sma50 and sma200: bits.append(f"50-day (${sma50:,.0f}) {'above' if gc else 'below'} the 200-day")
        if px and sma50: bits.append(f"price {'above' if px > sma50 else 'below'} the 50-day")
        if m3 is not None: bits.append(f"3-month return {m3*100:+.0f}%")
        if m12 is not None: bits.append(f"12-month return {m12*100:+.0f}%")
        return {"positive": positive, "negative": negative, "label": label, "detail": "; ".join(bits),
                "price": px, "sma50": sma50, "sma200": sma200, "ret_3m": m3, "ret_12m": m12,
                "rule": "positive = above 200-day AND 50-day above 200-day AND 3-month return ≥ 0"}
    except Exception:  # noqa: BLE001
        return {}


def _edgar_first_8k_mention(ticker, key, words, since):
    """Announcement date of a spin-off/divestiture: the earliest 8-K whose TEXT contains an
    announcement sentence naming the unit ("announced ... plan to separate Mobility"). A plain
    term co-occurrence search dated Mobility to a 2024 earnings release; the announcement was
    the 8-K of 2025-04-29."""
    import re as _re, html as _html
    try:
        import requests as _rq
        cik = _EDGAR_CIK.get(ticker.upper())
        if not cik or not key:
            return None
        hdr = {"User-Agent": _EDGAR_UA}
        cands = {}
        for w in words:
            try:
                r = _rq.get("https://efts.sec.gov/LATEST/search-index",
                            params={"q": f'"{key}" "{w}"', "ciks": f"{cik:010d}", "forms": "8-K", "dateRange": "custom", "startdt": since, "enddt": "2099-12-31"},
                            headers=hdr, timeout=25)
                for h in (r.json().get("hits") or {}).get("hits") or []:
                    d = (h.get("_source") or {}).get("file_date")
                    if d: cands[h.get("_id")] = d
            except Exception:
                continue
        sent_pat = _re.compile(rf"[^.]*\b(announc\w*|intend\w*|plan\w*|explor\w*|pursu\w*|decid\w*|approv\w*)[^.]*\b(separat\w*|spin[- ]?off|divest\w*|sale of)[^.]*\b{_re.escape(key)}\b[^.]*\."
                              rf"|[^.]*\b{_re.escape(key)}\b[^.]*\b(separat\w*|spin[- ]?off|divest\w*)[^.]*\b(announc\w*|intend\w*|plan\w*|explor\w*|pursu\w*|decid\w*|approv\w*)[^.]*\.", _re.I)
        for _id, d in sorted(cands.items(), key=lambda kv: kv[1])[:6]:
            try:
                adsh, fn = str(_id).split(":")
                t = _rq.get(f"https://www.sec.gov/Archives/edgar/data/{cik}/{adsh.replace('-', '')}/{fn}", headers=hdr, timeout=20).text
            except Exception:
                continue
            t = _html.unescape(_re.sub(r"\s+", " ", _re.sub(r"<[^>]+>", " ", t)))
            if sent_pat.search(t):
                return d
        return None
    except Exception:  # noqa: BLE001
        return None


def _edgar_deal_status(ticker, business, since):
    """Resolve whether an announced spin-off / divestiture / acquisition has CLOSED, from the
    company's own SEC filings (EDGAR full-text search + the filing text), instead of model
    recall. OSTTRA's $3.1B sale closed 2025-10-10; recall still had it 'pending' a year on.
    Returns {"status": "completed", "date" (closing date from the sentence, else filing date),
    "filed", "form", "snippet", "url"} or None (= unknown)."""
    import re as _re, html as _html
    try:
        import requests as _rq
    except ImportError:
        return None
    try:
        cik = _EDGAR_CIK.get(ticker.upper())
        if not cik:
            return None
        _generic = {"segment", "division", "business", "unit", "stake", "joint", "venture", "with", "post-trade",
                    "services", "data", "global", "group", "inc", "inc.", "corp", "the", "and", "firm", "company", "parent"}
        toks = [t.strip("(),.'\"") for t in str(business or "").split()]
        toks = [t for t in toks if len(t) > 3 and t.lower() not in _generic]
        if not toks:
            return None
        keys = []
        for k in (toks[0], max(toks, key=lambda t: (t.isupper(), len(t)))):
            if k not in keys:
                keys.append(k)
        hdr = {"User-Agent": _EDGAR_UA}
        _datepat = _re.compile(r"(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},\s+20\d\d")
        neg = _re.compile(r"(expected|anticipated|intend|plan|subject to|will be|to be)\s+(?:to\s+be\s+)?(?:completed|closed|consummated)", _re.I)
        for key in keys:
            pat = _re.compile(rf"((?:completed|closed|consummated|completion of|closing of)[^.]{{0,160}}\b{_re.escape(key)}\b|"
                              rf"\b{_re.escape(key)}\b[^.]{{0,200}}\b(?:was|were|has been|have been|had been|is now)?\s*(?:completed|closed|consummated|complete))", _re.I)
            try:
                r = _rq.get("https://efts.sec.gov/LATEST/search-index",
                            params={"q": f'"{key}"', "ciks": f"{cik:010d}", "dateRange": "custom", "startdt": since, "enddt": "2099-12-31"},
                            headers=hdr, timeout=25)
                hits = (r.json().get("hits") or {}).get("hits") or []
            except Exception:
                continue
            hits = sorted(hits, key=lambda h: (h.get("_source") or {}).get("file_date", ""), reverse=True)[:6]
            for h in hits:
                src = h.get("_source") or {}
                try:
                    adsh, fn = str(h.get("_id")).split(":")
                except ValueError:
                    continue
                url = f"https://www.sec.gov/Archives/edgar/data/{cik}/{adsh.replace('-', '')}/{fn}"
                try:
                    t = _rq.get(url, headers=hdr, timeout=20).text
                except Exception:
                    continue
                t = _html.unescape(_re.sub(r"\s+", " ", _re.sub(r"<[^>]+>", " ", t)))
                for m in pat.finditer(t):
                    sn = m.group(1)
                    if neg.search(sn):
                        continue
                    ctx = t[max(0, m.start() - 220): m.end() + 60]
                    dm_ = None
                    for dmm in _datepat.finditer(ctx):
                        try:
                            dm_ = datetime.strptime(dmm.group(0).replace(",", ""), "%B %d %Y").strftime("%Y-%m-%d")
                        except ValueError:
                            pass
                    return {"status": "completed", "date": dm_ or src.get("file_date"), "filed": src.get("file_date"),
                            "form": src.get("form"), "snippet": sn[:220].strip(), "url": url}
        return None
    except Exception:  # noqa: BLE001
        return None


_re_spin = __import__("re").compile(r"separation|spin-?off|distribution date", __import__("re").I)


def _restatement_from_consensus(eec, rev_ttm, rev_growth_yoy, actions, statements_through, edgar=None):
    """Consensus-vs-TTM restatement signal (see call site). Requires >=8 analysts, current-FY
    consensus revenue <95% of reported TTM, a next-FY consensus that is ALSO below TTM, and
    positive reported growth (a shrinking company legitimately has consensus below TTM)."""
    try:
        cur, nxt = eec.get("current_year_revenue"), eec.get("next_year_revenue")
        n = int(eec.get("analyst_count") or 0)
        g = rev_growth_yoy if rev_growth_yoy is not None else None
        if g is not None and abs(g) > 1.0: g = g / 100.0
        if not (cur and rev_ttm and rev_ttm > 0 and n >= 8):
            return None
        if not (cur < 0.95 * rev_ttm and (nxt is None or nxt < rev_ttm) and (g is None or g > 0.02)):
            return None
        gap = 1.0 - cur / rev_ttm
        # Name a unit only when a recalled spin-off/divestiture is the right SIZE for the gap
        # (share within 0.5x-2x of it) and is not an old, already-lapped deal; recall varies
        # run to run (one SPGI run pinned the 9% gap on a 2% unit sold in 2024).
        unit = None
        for a in (actions or []):
            if a.get("kind") not in ("spin_off", "divestiture"):
                continue
            sh = a.get("revenue_share")
            if not sh or not (0.5 * gap <= sh <= 2.0 * gap):
                continue
            eff = str(a.get("effective") or "")
            if len(eff) >= 4 and eff[:4].isdigit() and int(eff[:4]) < datetime.now().year - 1:
                continue
            if unit is None or abs(sh - gap) < abs((unit.get("revenue_share") or 0) - gap):
                unit = a
        # SEC 8-K Item 2.01 (completion of acquisition or disposition) filed after the window
        # start is hard evidence: it dates the event and often names it.
        _ek = None
        _win_start = (datetime.strptime(str(statements_through)[:10], "%Y-%m-%d") - timedelta(days=365)).strftime("%Y-%m-%d") if statements_through else "0000"
        for e in (edgar or []):
            if e.get("kind") == "completed_acquisition_or_disposition" and e.get("filed", "") >= _win_start:
                _ek = e; break
        # name the unit from the filing text when recall did not supply a size-consistent match
        _ename = None
        if unit is None and _ek and _ek.get("excerpt"):
            import re as _re2
            _mm = _re2.search(r"(?:separation|spin-off|spinoff|sale|disposition|divestiture) of (?:the )?([A-Z][A-Za-z0-9&'.\- ]{2,50}?)(?= from| to | business| segment| division|,|\.|\()", _ek["excerpt"], _re2.I)
            if _mm: _ename = _mm.group(1).strip()
        biz = unit["business"] if unit else (_ename or "a divested business")
        _what = (f"the {unit['kind'].replace('_', ' ')} of {unit['business']} having completed" if unit
                 else (f"the separation of {_ename} having completed" if _ename
                       else "a spin-off or divestiture having completed since the last reported quarter"))
        _conf = (f" This is confirmed by an 8-K (Item 2.01, completion of acquisition or disposition of assets) filed {_ek['filed']}"
                 + (f": \"{_ek['excerpt'][:220]}…\"" if _ek and _ek.get("excerpt") else ".")) if _ek else ""
        _eff_u = (unit or {}).get("effective")
        if _ek and (not _eff_u or len(str(_eff_u)) < 10):
            _eff_u = _ek.get("filed")                 # recalled "2026" → the 8-K's date
        _kind_e = ("spin_off" if (_ek and _re_spin.search(str(_ek.get("excerpt") or ""))) else "divestiture")
        return {"kind": (unit or {}).get("kind") or _kind_e, "business": biz,
                "effective": _eff_u,
                "revenue_share": (unit or {}).get("revenue_share") or round(gap, 2), "statements_through": str(statements_through)[:10],
                "basis": ("consensus + 8-K" if _ek else "consensus"), "edgar": (_ek and {"filed": _ek["filed"], "url": _ek["url"], "excerpt": _ek.get("excerpt")}),
                "consensus_current_fy_usd_b": round(cur / 1e9, 2), "reported_ttm_usd_b": round(rev_ttm / 1e9, 2),
                "note": (f"Sell-side consensus revenue for the current fiscal year (${cur/1e9:,.1f}B, {n} analysts) is "
                         f"{gap*100:.0f}% below reported trailing-twelve-month revenue (${rev_ttm/1e9:,.1f}B) although reported revenue "
                         f"grew {g*100:.0f}% year over year — analysts are modelling a smaller continuing company, consistent with "
                         f"{_what}; reported TTM revenue, margins and multiples still include the divested business." + _conf)}
    except Exception:  # noqa: BLE001
        return None


def _nominate_peers(ticker, name, industry, description, call_json, hint=None):
    """Business-model comparables nominated by a model and later verified in the panel. Sector +
    market-cap matching put banks and insurers next to a ratings/index company (SPGI)."""
    try:
        out = call_json(
            "You are an equity analyst choosing valuation comparables. Name the 8 closest US-listed public "
            "companies by BUSINESS MODEL and end-market (same economics: how they earn revenue, margin "
            "structure, growth drivers) — not merely the same sector label. Prefer larger, widely covered "
            "names. STRICT JSON only: {\"peers\": [\"TICK\", ...]}",
            f"Company: {name or ticker} (ticker {ticker}). Industry: {industry or '?'}. "
            f"Description: {(description or '')[:500]}" + (f"\nNOTE: {hint}" if hint else ""))
        peers = [str(x).upper().strip() for x in ((out or {}).get("peers") or []) if isinstance(x, str)]
        return [x for x in peers if 1 <= len(x) <= 6 and x != ticker.upper()][:10]
    except Exception:  # noqa: BLE001
        return []


def _corporate_events_brief(ticker, name, call_json):
    """Model-recalled material events (e.g. a CEO succession announced months ago) — the live
    news feed only covers the last few days, so slow-burn corporate context never reaches the
    debate without this. Returns a list of {date, event} or []."""
    try:
        out = call_json(_CORP_EVENTS_SYSTEM, f"Company: {name or ticker} (ticker {ticker}). Today: {datetime.now().strftime('%Y-%m')}.")
        ev = (out or {}).get("events") or []
        good = []
        for e in ev[:5]:
            if isinstance(e, dict) and e.get("event"):
                good.append({"date": str(e.get("date") or "")[:7], "event": str(e["event"])[:160]})
        return good
    except Exception:  # noqa: BLE001
        return []


def _peer_comps(ticker, sector, mcap, n=8, prefer=None, exclude_reits=False):
    """Live peer comparables from the weekly Sharadar panel: same sector, nearest by log
    market cap. Returns {peers:[...], subject:{...}, percentiles:{metric: pct}} or None.
    Replaces the static sector-median table as the debate's relative-value context."""
    try:
        import glob as _g
        import damodaran_portfolio as _dp
        _cols = ["week_ending", "ticker", "sector", "marketcap", "pe", "ev_ebitda",
                 "sales_yield", "fcf_yield", "operating_margin", "gross_margin",
                 "rev_growth_yoy", "net_debt_ebitda", "dividend_yield", "roe", "beta_mkt"]
        _full = sorted(_g.glob(f"{_dp.PANEL}/*.parquet")) if os.path.isdir(_dp.PANEL) else []
        if _full:
            df = pd.read_parquet(_full[-1], columns=_cols)
        else:
            # Space: the full panel lives on the research machine; a slim weekly extract of
            # the peer metrics is vendored alongside the engine.
            _slim = os.path.join(os.path.dirname(os.path.abspath(__file__)), "peer_metrics.parquet")
            df = pd.read_parquet(_slim)
        cur = df[df.week_ending == df.week_ending.max()].copy()
        cur = cur[cur.marketcap > 0]
        me = cur[cur.ticker == ticker.upper()]
        sec = (me.sector.iloc[0] if len(me) else None) or sector
        _secn = str(sec or "").strip().lower()
        # 1) business-model peers nominated by the model, verified present in the panel
        _pref = [str(x).upper() for x in (prefer or []) if x and str(x).upper() != ticker.upper()]
        pref_rows = cur[cur.ticker.isin(_pref)].copy() if _pref else cur.iloc[0:0].copy()
        if len(pref_rows):
            _order = {t: i for i, t in enumerate(_pref)}
            pref_rows["dist"] = pref_rows.ticker.map(_order).astype(float)
            pref_rows = pref_rows.sort_values("dist").head(n)
        # 2) fill with same-sector names nearest in market cap
        pool = cur[(cur.sector.astype(str).str.strip().str.lower() == _secn) & (cur.ticker != ticker.upper())
                   & (~cur.ticker.isin(pref_rows.ticker))].copy()
        _reit_note = ""
        if exclude_reits:
            # the panel has no industry column: REITs are the dividend-paying, high-operating-margin
            # names in the sector; services firms (CBRE, JLL, BEKE, CSGP) pay little/no dividend
            # and run single-digit to low-teens operating margins
            _dy = pd.to_numeric(pool.dividend_yield, errors="coerce").fillna(0.0)
            _om = pd.to_numeric(pool.operating_margin, errors="coerce")
            pool = pool[(_dy <= 0.005) & (_om.isna() | (_om < 0.15))].copy()
            _reit_note = " (REITs excluded: the subject is a real-estate services company)"
        if len(pool) < 3 and len(pref_rows) < 3:
            return None
        ref = float(me.marketcap.iloc[0]) if len(me) else float(mcap or (pool.marketcap.median() if len(pool) else 0) or 1e9)
        pool["dist"] = (np.log(pool.marketcap) - np.log(ref)).abs() + 1000.0   # always after nominated peers
        # enough verified business-model comparables: no same-sector fill (a bank and an auto insurer filled SPGI's list
        # to eight); the market-cap fill applies only when fewer than three nominated peers are in the panel
        _fill = 0 if len(pref_rows) >= 3 else max(0, n - len(pref_rows))
        peers = pd.concat([pref_rows, pool.sort_values("dist").head(_fill)])
        _basis = (f"{len(pref_rows)} business-model comparables nominated by the model and verified in the Sharadar panel"
                  + (f", plus {len(peers) - len(pref_rows)} same-sector names nearest in market cap{_reit_note}" if len(peers) > len(pref_rows) else "")
                  if len(pref_rows) else f"{len(peers)} same-sector names nearest in market cap{_reit_note}")
        def row(r):
            def v(x, mult=1.0, cap=None):
                try:
                    x = float(x)
                    if x != x: return None
                    x *= mult
                    return None if (cap and abs(x) > cap) else x
                except Exception: return None
            ps = v(r.get("sales_yield"))
            return {"ticker": r["ticker"], "mcap_b": v(r.get("marketcap"), 1e-9),
                    "source": "NOMINATED" if r["ticker"] in set(pref_rows.ticker) else "SECTOR_FILL",
                    "pe": v(r.get("pe"), cap=300), "ev_ebitda": v(r.get("ev_ebitda"), cap=200),
                    "ps": (round(1.0 / ps, 2) if ps and ps > 0 else None),
                    "fcf_yield": v(r.get("fcf_yield")), "op_margin": v(r.get("operating_margin")),
                    "rev_growth": v(r.get("rev_growth_yoy")), "net_debt_ebitda": v(r.get("net_debt_ebitda"), cap=50),
                    "div_yield": v(r.get("dividend_yield")), "roe": v(r.get("roe"))}
        plist = [row(r) for _, r in peers.iterrows()]
        subj = row(me.iloc[0]) if len(me) else None
        if subj and mcap and subj.get("mcap_b") and 0.5 < (mcap / 1e9) / subj["mcap_b"] < 2.0:
            subj["mcap_b"] = mcap / 1e9          # live market cap (share count changes after a spin)
        pct = {}
        if subj:
            for k in ("pe", "ev_ebitda", "ps", "op_margin", "rev_growth", "fcf_yield"):
                vals = [x[k] for x in plist if x.get(k) is not None]
                if subj.get(k) is not None and len(vals) >= 3:
                    pct[k] = round(sum(1 for x in vals if x <= subj[k]) / len(vals) * 100)
        return {"sector": sec, "as_of": str(cur.week_ending.max().date()), "peers": plist,
                "subject": subj, "percentiles": pct, "basis": _basis,
                "nominated": [t for t in _pref if t in set(pref_rows.ticker)]}
    except Exception:  # noqa: BLE001
        return None


def _risk_card(data, overview):
    """PM sizing card from the daily series: realized vol, drawdown, distance from 52w high,
    liquidity, correlation to SPY — plus the overview beta."""
    try:
        d = data.get("daily")
        if d is None or len(d) < 60:
            return None
        px = d["adj_close"].dropna()
        last = px.index.max()
        y = px[px.index >= last - pd.Timedelta(days=365)]
        r = np.log(y).diff().dropna()
        vol = float(r.std() * np.sqrt(252))
        peak = y.cummax(); dd = float((y / peak - 1.0).min())
        hi52 = float(y.max()); dist_hi = float(y.iloc[-1] / hi52 - 1.0)
        dv = None
        if "volume" in d.columns:
            tail = d[d.index >= last - pd.Timedelta(days=45)]
            dv = float((tail["close"] * tail["volume"]).tail(30).mean())
        corr = None
        spy = data.get("spy")
        if spy is not None and len(spy) > 60:
            sp = np.log(spy["adj_close"]).diff().dropna()
            j = pd.concat([r.rename("a"), sp.rename("b")], axis=1).dropna()
            if len(j) > 60:
                corr = float(j["a"].corr(j["b"]))
        return {"beta": safe_float(overview.get("Beta")), "vol_1y": round(vol, 3), "max_drawdown_1y": round(dd, 3),
                "dist_52w_high": round(dist_hi, 3), "avg_dollar_volume_30d": dv,
                "corr_spy_1y": (round(corr, 2) if corr is not None else None), "as_of": str(last.date())}
    except Exception:  # noqa: BLE001
        return None


def _wiki_leadership(company_name):
    """Current leadership from the Wikipedia infobox (live, grounded — catches announced CEO
    transitions that both the few-day news feed and model recall miss; AAPL/Ternus case).
    Follows redirects ('Broadcom Inc.' -> 'Broadcom') and parses nested templates ({{ubl|...}})
    brace-aware — the naive version failed on 5 of 6 large caps."""
    try:
        import urllib.request as _u, urllib.parse as _up, ssl as _ssl
        try:
            import certifi as _ce
            _ctx = _ssl.create_default_context(cafile=_ce.where())
        except Exception:
            _ctx = _ssl.create_default_context()
        _ua = {"User-Agent": "CrucibleAnalyst/1.0 (https://huggingface.co/spaces/s-ttp/crucible; research tool)"}
        def _get(url):
            try:
                import requests as _rq
                r = _rq.get(url, headers=_ua, timeout=12)
                r.raise_for_status()
                return r.text
            except Exception:
                req = _u.Request(url, headers=_ua)
                with _u.urlopen(req, timeout=12, context=_ctx) as r:
                    return r.read().decode()
        q = _up.quote(str(company_name or ""))
        sr = json.loads(_get(f"https://en.wikipedia.org/w/api.php?action=opensearch&search={q}&limit=1&format=json"))
        if not sr[1]:
            return None
        t = _up.quote(sr[1][0])
        w = json.loads(_get(f"https://en.wikipedia.org/w/api.php?action=query&redirects=1&titles={t}"
                            f"&prop=revisions&rvprop=content&rvslots=main&rvsection=0&format=json"))
        page = list(w["query"]["pages"].values())[0]
        text = page.get("revisions", [{}])[0].get("slots", {}).get("main", {}).get("*", "")
        i = re.search(r'\|\s*key_people\s*=', text)
        if not i:
            return None
        # brace-aware scan: the value often IS a template ({{ubl|A|B}}) whose own '|' lines
        # would otherwise terminate a naive regex after the first name.
        j = i.end(); depth = 0; out = []
        while j < len(text):
            two = text[j:j + 2]
            if two == "{{":
                depth += 1; out.append("{{"); j += 2; continue
            if two == "}}":
                depth -= 1; out.append("}}"); j += 2; continue
            ch = text[j]
            if depth == 0 and ch == "\n" and text[j + 1:j + 2] == "|":
                break
            out.append(ch); j += 1
        raw = "".join(out)
        raw = re.sub(r'\{\{\s*cite[^}]*\}\}', ' ', raw, flags=re.I | re.S)          # drop citations
        raw = re.sub(r'<ref[^>]*>.*?</ref>|<ref[^>]*/>', ' ', raw, flags=re.I | re.S)
        raw = re.sub(r'<br\s*/?>|<[^>]+>', ' | ', raw)
        raw = re.sub(r'\{\{(?:ubl|unbulleted list|plainlist|hlist|small|nowrap)\s*\|?', ' ', raw, flags=re.I)
        raw = re.sub(r'\[\[[^\]|]*\|', '', raw)                                       # [[Target|label]] -> label
        raw = re.sub(r'\{\{|\}\}|\[\[|\]\]|\*', ' ', raw)
        raw = re.sub(r'\s*\|\s*', ' | ', raw)
        raw = re.sub(r'(\s*\|\s*){2,}', ' | ', raw)
        raw = re.sub(r'\s+', ' ', raw).strip(" |")
        return raw[:300] if raw else None
    except Exception:  # noqa: BLE001
        return None




def main():
    parser = argparse.ArgumentParser(
        description="1-Year Stock Price Predictor — Fundamental + ML + optional LLM annotation",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python3 predict_yearly.py AAPL
  python3 predict_yearly.py MSFT --llm
        """,
    )
    parser.add_argument("ticker", help="Stock ticker (e.g. AAPL)")
    parser.add_argument("--no-cache", action="store_true", help="Force fresh data")
    parser.add_argument("--llm", action="store_true", help="(default on) kept for compatibility")
    parser.add_argument("--no-llm", action="store_true", help="Skip the 4-role LLM debate")
    parser.add_argument("--verbose", action="store_true")
    parser.add_argument("--pdf", action="store_true", help="Generate a PDF report with AI-written summary")
    parser.add_argument("--json", action="store_true", help="Export the full report data to TICKER_report.json")

    args = parser.parse_args()
    ticker = args.ticker.upper()
    # 4-role debate is DEFAULT for single-stock deep diligence (this is where the panel lives now)
    args.llm = not args.no_llm

    print(f"\n{BOLD}{MAGENTA}{'═' * 68}{RESET}")
    print(f"{BOLD}{MAGENTA}  1-Year Price Forecast — {ticker}{RESET}")
    print(f"{BOLD}{MAGENTA}  {datetime.now().strftime('%Y-%m-%d %H:%M')}  |  Fundamental + ML + LLM Intelligence{RESET}")
    print(f"{BOLD}{MAGENTA}{'═' * 68}{RESET}\n")

    # ── Step 1: Collect Data ──
    print(f"{BOLD}[1/5] Collecting Data (17 API endpoints){RESET}")
    all_data = fetch_all_data_yearly(ticker, years=15, no_cache=args.no_cache)
    # ── Canonical layer (deterministic): who is this issuer, which filings, which model ──
    try:
        import canonical as _cn
        _subm = _cn.edgar_submissions(ticker)
        if _subm.get("cik"):
            _EDGAR_CIK[ticker.upper()] = _subm["cik"]; _EDGAR_SUB[_subm["cik"]] = _subm["sub"]
        all_data["xbrl"] = _cn.edgar_companyfacts(_subm.get("cik"))
        all_data["issuer"] = _cn.classify_issuer(ticker, all_data.get("overview"), _subm, all_data.get("xbrl"))
        all_data["valuation_framework"] = _cn.select_valuation_framework(all_data["issuer"], all_data.get("overview"))
        _is_ = all_data["issuer"]; _fw_ = all_data["valuation_framework"]
        print(f"  [issuer] {_is_['filer_type']} · {_is_['issuer_class']} ({_is_['class_basis']}) · {_is_['accounting_standard']} · reports {_is_['reporting_currency']}, quoted {_is_['quote_currency']}"
              + (" · ADR" if _is_.get("is_adr") else "") + (" · MIXED financial" if _is_.get("mixed_financial") else ""))
        print(f"  [framework] {_fw_['framework']} — {_fw_['valuation_framework_reason'][:140]}")
        # ── applicability policy + upstream canonicalisation (one concept per idea, quarantine the rest) ──
        import policy as _pol
        all_data["policy"] = _pol.ApplicabilityPolicy.for_issuer(all_data["issuer"])
        _rej = _pol.canonicalize_statements(all_data, all_data["issuer"])
        if _rej:
            print(f"  [canonical] revenue concept {all_data['canonical_concepts']['revenue']}: {_rej['reason'][:110]} ({_rej.get('reports_rewritten', 0)} reports rewritten, gross series quarantined)")
        elif (all_data.get("canonical_concepts") or {}).get("revenue") == "UNVERIFIED":
            print(f"  {YELLOW}[canonical] revenue concept UNVERIFIED for this {all_data['issuer']['issuer_class']} — revenue forecasts and consensus comparisons will be blocked{RESET}")
        # primary-source facts the models and gates need: segment facts (annual-report XBRL instance),
        # bank capital/credit facts (company facts), ADS ratio and officers (20-F) for foreign issuers
        try:
            all_data["segment_facts"] = _cn.edgar_segment_facts(_subm.get("cik"), _subm.get("sub"))
            all_data["segment_mix"] = _cn.edgar_segment_mix(_subm.get("cik"), _subm.get("sub"))
            _smx = all_data["segment_mix"] or {}
            if _smx.get("segments"):
                print(f"  [segments/filed] {_smx.get('form')} {_smx.get('period_start')}→{_smx.get('period_end')}: " + "; ".join(
                    f"{g['name']} {g['revenue_share']*100:.0f}%" + (f" @{g['margin']*100:.0f}% op" if g.get("margin") is not None else "")
                    for g in _smx["segments"][:5]))
            _sfx = all_data["segment_facts"]
            import segments as _sgm
            all_data["segments_canonical"] = _sgm.canonicalize(_smx, _sfx)
            _scn = all_data["segments_canonical"]
            print(f"  [segments/canonical] {_scn['status']}: {len(_scn.get('segments') or [])} segment(s)"
                  + (f", {len(_scn['aliases'])} alias member(s) merged" if _scn.get("aliases") else "")
                  + (f", Σ = {_scn['coverage']:.1%} of consolidated revenue" if _scn.get("coverage") else "")
                  + ("" if _scn["status"] in ("RECONCILED", "SINGLE_SEGMENT", "UNAVAILABLE") else f" — {str(_scn.get('reason'))[:140]}"))
            if _sfx.get("segments"):
                print(f"  [segments/SEC] FY{str(_sfx.get('fy_end'))[:4]}: " + "; ".join(f"{g['name']} ({g['kind']}) rev ${(g['revenue'] or 0)/1e9:,.1f}B" + (f", profit ${g['profit']/1e9:,.1f}B" if g.get('profit') is not None else "") for g in _sfx['segments'][:5])
                      + (f"; by line: " + ", ".join(f"{l['name']} ${l['revenue']/1e9:,.1f}B" for l in _sfx.get('revenue_by_line', [])[:4]) if _sfx.get("revenue_by_line") else ""))
            # company guidance from its own earnings release (8-K Item 2.02 ex-99.1; 6-K for a foreign issuer)
            try:
                import guidance as _gd, damodaran_portfolio as _dpg
                all_data["company_guidance"] = _gd.company_guidance(_subm.get("sub"), _subm.get("cik"), _cn._get,
                                                                    lambda sy, us: _dpg._call_claude_json(sy, us, temperature=0.0, max_tokens=8000))
                _cg = all_data["company_guidance"]
                print(f"  [guidance] {_cg['status']}" + (f": {len(_cg['metrics'])} verified row(s) from {(_cg.get('source') or {}).get('form')} "
                                                         f"filed {(_cg.get('source') or {}).get('filed')}" if _cg.get("metrics") else f" — {_cg.get('reason')}")
                      + (f"; {len(_cg['rejected'])} unverifiable row(s) rejected" if _cg.get("rejected") else ""))
                _cov = (_cg.get("diagnostics") or {})
                if _cov.get("candidate_rows") is not None:
                    print("  [guidance] coverage: " + "; ".join(
                        f"{x['concept']} → {((_cg.get('concept_states') or {}).get(x['concept']) or {}).get('state')}"
                        for x in _cov["candidate_rows"]) + f" | coverage pass: {(_cov.get('coverage_pass') or {}).get('requested')}")
            except Exception as _gde:  # noqa: BLE001
                all_data["company_guidance"] = {"status": "UNAVAILABLE", "reason": f"guidance retrieval failed: {str(_gde)[:80]}", "metrics": []}
            if all_data["issuer"].get("is_financial"):
                all_data["bank_facts"] = _cn.bank_facts(_subm.get("cik"))
                _bfx = all_data["bank_facts"]
                print("  [bank/SEC] " + ", ".join(f"{k} {'ok' if isinstance(v, dict) and v.get('value') is not None else 'n/a'}" for k, v in _bfx.items() if k != "source"))
            if all_data["issuer"].get("is_fpi"):
                all_data["ads_ratio"] = _cn.edgar_ads_ratio(_subm.get("cik"), _subm.get("sub"))
                all_data["officers_20f"] = _cn.edgar_20f_officers(_subm.get("cik"), _subm.get("sub"))
                if all_data["ads_ratio"].get("ratio"):
                    # set on the issuer NOW: every per-share metric is built on the quoted basis
                    all_data["issuer"]["is_adr"] = True
                    all_data["issuer"]["adr_ratio"] = all_data["ads_ratio"]["ratio"]
                    all_data["issuer"]["adr_source"] = f"SEC 20-F filed {all_data['ads_ratio'].get('filed')}"
                    print(f"  [20-F] ADS ratio {all_data['ads_ratio']['ratio']}:1 — \"{all_data['ads_ratio']['sentence'][:90]}…\"")
                if all_data["officers_20f"]: print("  [20-F] officers: " + "; ".join(f"{k} {v['name']}" for k, v in all_data["officers_20f"].items()))
        except Exception as _px:  # noqa: BLE001
            print(f"  {YELLOW}[primary-source] extras failed: {str(_px)[:80]}{RESET}")
        print(f"  [policy] {all_data['policy'].issuer_class}: engine {all_data['policy'].valuation_engine}; pricing methods {list(all_data['policy'].allowed_pricing_methods)}"
              + (f"; scorecard rows disabled: {len(all_data['policy'].disallowed_scorecard_rows)}" if all_data['policy'].disallowed_scorecard_rows else ""))
    except Exception as _ie:  # noqa: BLE001
        print(f"  {YELLOW}[issuer] classification unavailable: {str(_ie)[:80]}{RESET}")

    daily = all_data["daily"]
    spy_df = all_data.get("spy")
    current_price = daily["adj_close"].iloc[-1]
    # Prefer real-time global quote over potentially stale cached daily close
    _gq_price = safe_float((all_data.get("quote") or {}).get("price"))
    if _gq_price and _gq_price > 0:
        current_price = _gq_price

    # ── SOURCE RECONCILIATION (before any model): SEC filings vs provider statements, one basis ──
    import pipeline as _pl
    _prun = _pl.Run(ticker)
    _final_path = f"{ticker}_final.json"
    try:
        import reconcile as _rcn
        _rh = _rcn.reconcile_historicals(all_data, cik=(locals().get("_subm") or {}).get("cik"),
                                         fx_to_usd=safe_float(all_data.get("fx_usd")))
        print(f"  [reconcile] {_rh.get('status')} — {_rh.get('basis')}; {len(_rh.get('overrides') or [])} provider value(s) replaced"
              + (f"; unverified quarters: {', '.join(_rh['unverified_quarters'])}" if _rh.get("unverified_quarters") else ""))
    except Exception as _rce:  # noqa: BLE001
        print(f"  {YELLOW}[reconcile] failed: {str(_rce)[:100]}{RESET}")
        all_data["reconciled_historicals"] = {"status": "NO_STATEMENTS", "basis": f"reconciliation failed: {str(_rce)[:80]}"}
    try:
        _shr = _rcn.reconcile_shares(all_data, current_price)
        if _shr.get("changed"):
            print(f"  [shares] {_shr['basis']}: {(_shr['canonical'] or 0)/1e9:,.2f}B (provider {(_shr.get('provider') or 0)/1e9:,.2f}B)")
    except Exception as _she:  # noqa: BLE001
        print(f"  {YELLOW}[shares] reconciliation failed: {str(_she)[:80]}{RESET}")
    try:
        import damodaran_data as _ddr
        print(f"  [damodaran] implied ERP {_ddr.refresh_erp_if_stale()}")
    except Exception:  # noqa: BLE001
        pass
    # PERIMETER (a data fact, decided before the data stage): a completed separation leaves the trailing
    # statements on the old perimeter while consensus models the continuing company. Needs only
    # consensus + the reconciled profile + statements + SEC 8-Ks.
    try:
        _ovs0 = all_data.get("overview") or {}
        _stq00 = (((all_data.get("income") or {}).get("quarterlyReports") or [{}])[0] or {}).get("fiscalDateEnding")
        all_data["edgar_actions"] = _edgar_actions(ticker)
        if all_data["edgar_actions"]:
            print("  [edgar] " + " | ".join(f"8-K {e['filed']} items {','.join(e['items'])}" for e in all_data["edgar_actions"][:4]))
        all_data["restatement"] = _restatement_from_consensus(
            all_data.get("earnings_estimates_consensus") or {}, safe_float(_ovs0.get("RevenueTTM")),
            safe_float(_ovs0.get("QuarterlyRevenueGrowthYOY")), [], _stq00, edgar=all_data["edgar_actions"])
        if all_data["restatement"]:
            print(f"  {YELLOW}[restatement] {all_data['restatement']['note']}{RESET}")
    except Exception:  # noqa: BLE001
        all_data["restatement"] = None
    # one typed perimeter adjustment for the separated business (same period as the trailing statements), the
    # continuing-company story base built from it, and the segment set of the forward perimeter
    try:
        import story_base as _sbm, corporate_actions as _cam
        _rst0 = all_data.get("restatement")
        if _rst0 and _rst0.get("business"):
            _aid0 = _cam.action_id(ticker, _rst0.get("kind"), _rst0.get("business"))
            all_data["perimeter_adjustment"] = _sbm.perimeter_adjustment(
                _aid0, _rst0.get("business"), all_data.get("segments_canonical"), all_data.get("segment_facts"),
                all_data.get("reconciled_historicals") or {})
            _pa = all_data["perimeter_adjustment"]
            print(f"  [perimeter] {_aid0}: {_pa.get('method')} — {_rst0.get('business')} "
                  + (f"{_pa['revenue_share']*100:.1f}% of revenue at {_pa['operating_margin']*100:.1f}% operating margin "
                     f"({_pa.get('period_start')}→{_pa.get('period_end')}, {_pa.get('quality')})" if _pa.get("revenue_share") is not None else _pa.get("reason", "")))
        all_data["story_base"] = _sbm.story_base(all_data.get("reconciled_historicals") or {}, all_data.get("perimeter_adjustment"),
                                                 (all_data.get("earnings_estimates_consensus") or {}).get("current_year_revenue"))
        if all_data.get("perimeter_adjustment", {}).get("segment_ids"):
            import segments as _sgp
            all_data["segments_forward"] = _sgp.for_perimeter(all_data.get("segments_canonical"),
                                                              all_data["perimeter_adjustment"]["segment_ids"],
                                                              all_data["perimeter_adjustment"]["action_id"])
    except Exception as _pbe:  # noqa: BLE001
        print(f"  {YELLOW}[perimeter] adjustment unavailable: {str(_pbe)[:100]}{RESET}")
    try:
        all_data["damodaran_inputs"] = _damodaran_inputs(ticker, all_data, ((locals().get("_subm") or {}).get("sub") or {}).get("sic"))
        _di = all_data["damodaran_inputs"]
        if _di.get("available"):
            print(f"  [damodaran] industry {_di.get('industry')} ({_di.get('industry_match')}), bottom-up beta {(_di.get('beta') or 0):.2f}, "
                  f"ERP {(_di.get('erp') or 0)*100:.2f}% ({_di.get('erp_basis')}; {_di.get('erp_month')}), industry sales/capital {_di.get('industry_sales_to_capital')}, "
                  f"stable cost of capital {(_di.get('stable_cost_of_capital') or 0)*100:.2f}% ({_di.get('beta_basis')})")
    except Exception as _dde:  # noqa: BLE001
        all_data["damodaran_inputs"] = {"available": False, "reason": str(_dde)[:120]}

    # ── Step 2: Build Point-in-Time Features ──
    print(f"\n{BOLD}[2/5] Building Point-in-Time Features{RESET}")

    print(f"  PIT fundamental features...", end=" ", flush=True)
    pit_fund = build_pit_fundamental_features(daily.index, all_data)
    if pit_fund is not None:
        n_fund_cols = len([c for c in pit_fund.columns if c.startswith("pit_")])
        n_rows = pit_fund.dropna(how="all").shape[0]
        print(f"{GREEN}{n_fund_cols} metrics × {n_rows} days{RESET}")
    else:
        print(f"{DIM}unavailable (no quarterly data){RESET}")

    print(f"  PIT macro features...", end=" ", flush=True)
    pit_macro = build_pit_macro_features(daily.index, all_data)
    n_macro_cols = len([c for c in pit_macro.columns if c.startswith("macro_")])
    print(f"{GREEN}{n_macro_cols} metrics{RESET}")

    print(f"  Intraday volume profile...", end=" ", flush=True)
    vp = fetch_intraday_volume_profile(ticker, no_cache=args.no_cache)
    if vp:
        print(f"{GREEN}OK{RESET}")
    else:
        print(f"{DIM}skip{RESET}")

    ml_extra = {
        "vix_df": all_data.get("vix_df"),
        "shares_outstanding": all_data.get("shares_outstanding"),
        "historical_options": all_data.get("historical_options"),
        "insider": all_data.get("insider"),
        "volume_profile": vp or {},
        "short_interest_pct_float": all_data.get("short_interest_pct_float"),
        "dividends": all_data.get("dividends"),
        "put_call_ratio_hist": all_data.get("put_call_ratio_hist"),
        "put_call_ratio_current": all_data.get("put_call_ratio_current"),
        "earnings_estimates_consensus": all_data.get("earnings_estimates_consensus"),
        "fundamentals_data": {
            "earnings": all_data.get("earnings", {}),
            "overview": all_data.get("overview", {}),
        },
    }

    # ── Step 3: Run Analysis Models ──
    print(f"\n{BOLD}[3/5] Running Analysis Models{RESET}")

    print(f"  Technical analysis (yearly)...", end=" ", flush=True)
    tech = analyze_technicals_yearly(daily, spy_df)
    all_data["momentum_canon"] = _momentum_canon(current_price, tech, all_data.get("overview"))
    try:
        import canonical as _cnd
        _mpre = _cnd.build_metrics(all_data, None, None, current_price)
        _iss = all_data.get("issuer") or {}
        if _iss.get("is_fpi") and not _iss.get("adr_ratio"):
            _inf = _cnd.infer_ads_ratio(all_data, _mpre)
            if _inf:
                _iss["is_adr"] = True; _iss["adr_ratio"] = _inf
                _iss["adr_source"] = f"inferred: {_inf} ordinary shares per ADS (SEC per-share data vs the quoted per-ADS figures)"
                _mpre = _cnd.build_metrics(all_data, None, None, current_price)
    except Exception as _mpe:  # noqa: BLE001
        print(f"  {YELLOW}[data] metric inputs failed: {str(_mpe)[:100]}{RESET}")
        _mpre = {}
    _dv = _prun.data_stage(all_data, current_price, price_as_of=(all_data.get("quote") or {}).get("latest trading day"),
                          metrics_pre=_mpre, tech=tech)
    print(f"  [data] {'PASS' if _dv.passed else RED + 'FAIL' + RESET} — " + ", ".join(f"{c.id} {c.status}" for c in _dv.checks))
    if not _dv.passed:
        for c in _dv.blocking:
            print(f"      !! {c.id} {c.name}: {c.detail[:160]}")
        _prun.write(_final_path)
        print(f"  [final] WITHHELD at data validation — no forecast, story or analyst panel was run. Saved {_final_path}")
        return
    tc = GREEN if tech["expected_return"] > 0 else RED
    print(f"{tc}{tech['expected_return']*100:+.2f}%{RESET} (conf:{tech['confidence']*100:.0f}%)")

    print(f"  Fundamental scorecard...", end=" ", flush=True)
    fund = analyze_fundamentals(all_data)
    try:
        import policy as _polf
        if all_data.get("policy") and fund:
            fund, _pa = _polf.apply_policy_to_scorecard(fund, all_data["policy"], all_data, current_price)
            if _pa["removed"] or _pa["added"]:
                print(f"  [policy] scorecard: {len(_pa['removed'])} generic row(s) disabled for a {all_data['policy'].issuer_class}; {len(_pa['added'])} issuer-specific row(s) added")
    except Exception as _pe:  # noqa: BLE001
        print(f"  {YELLOW}[policy] scorecard enforcement failed: {str(_pe)[:80]}{RESET}")
    if fund:
        # Widen return mapping for yearly horizon: ±20% instead of ±10%
        base_ret = (fund["score"] - 0.5) * 0.36  # range: -18% to +18%
        analyst_bonus = 0.0
        s = fund["signals"]
        if "analyst_upside" in s:
            analyst_bonus += np.clip(s["analyst_upside"] * 0.25, -0.12, 0.12)
        if "beat_rate" in s:
            analyst_bonus += np.clip((s["beat_rate"] - 0.5) * 0.08, -0.04, 0.04)
        # Scale analyst bonus by scorecard quality so a poor score can't be
        # fully overridden by analyst consensus (0% score → 20% of bonus,
        # 50% score → 60% of bonus, 100% score → full bonus)
        quality_scale = 0.2 + 0.8 * fund["score"]
        analyst_bonus *= quality_scale
        fund["expected_return"] = np.clip(base_ret + analyst_bonus, -0.40, 0.60)
        fc = GREEN if fund["expected_return"] > 0 else RED
        print(f"{fc}{fund['expected_return']*100:+.2f}%{RESET} (score:{fund['total_points']:.0f}/{fund['max_points']:.0f} conf:{fund['confidence']*100:.0f}%)")
    else:
        print(f"{DIM}unavailable{RESET}")

    print(f"  Sentiment analysis...", end=" ", flush=True)
    sent = analyze_sentiment(all_data, ticker)
    if sent:
        # Dampen sentiment for yearly (less predictive)
        sent["expected_return"] *= 0.5
        sent["confidence"] *= 0.7
        sc_color = GREEN if sent["expected_return"] > 0 else RED
        print(f"{sc_color}{sent['expected_return']*100:+.2f}%{RESET} (conf:{sent['confidence']*100:.0f}%)")
    else:
        print(f"{DIM}unavailable{RESET}")

    print(f"\n{BOLD}[3b/5] Fundamental ML Prediction (Stage 1 + 2){RESET}")
    print(f"  Predicting quarterly fundamentals...", end=" ", flush=True)
    pred_fund_result, fair_val_result = analyze_fundamental_ml_yearly(all_data, current_price,
                                                                      state=getattr(_prun, "state", None))
    # ── Damodaran "stories to numbers": story-driven FCFF for growth-stage names ──────
    # His growth valuations are built top-down (TAM × share → revenue path → target margin
    # → reinvestment via sales-to-capital → FCFF), not grown from a point EPS forecast.
    # The persona tells the story in numbers; story_dcf does the math deterministically.
    # Mature-stable/decline names keep the mechanical DCF (his own inputs there come from
    # delivered numbers, so the story adds little).
    story_val_result = None
    _CRUCIBLE_STORY_STAGE = None
    try:
        _ovs = (all_data.get("overview") or {})
        # ── Young IPOs (SpaceX: 49 days of prices, 4 quarters of statements) defeat the multiples
        #    engine (pred_fund=None → fair_val=None), but the STORY build only needs revenue,
        #    shares and price — and it is precisely Damodaran's tool for a young company.
        #    Derive what the overview lacks from the raw statements, and stage from trailing
        #    data when the ML forecast is unavailable.
        _iq_all = ((all_data.get("income") or {}).get("quarterlyReports") or [])
        _rev_q = [safe_float(q.get("totalRevenue")) for q in _iq_all[:4]]
        _rev_ttm_stmt = sum(v for v in _rev_q if v) if any(_rev_q) else None
        _opi_q = [safe_float(q.get("operatingIncome")) for q in _iq_all[:4]]
        _opi_ttm = sum(v for v in _opi_q if v is not None) if any(v is not None for v in _opi_q) else None
        _mcap_ov = safe_float(_ovs.get("MarketCapitalization"))
        _shares_ov = safe_float(_ovs.get("SharesOutstanding")) or (
            (_mcap_ov / current_price) if (_mcap_ov and current_price) else None)
        _rev_ov = safe_float(_ovs.get("RevenueTTM")) or _rev_ttm_stmt
        _opm_ov = safe_float(_ovs.get("OperatingMarginTTM"))
        if _opm_ov is None and _opi_ttm is not None and _rev_ov:
            _opm_ov = _opi_ttm / _rev_ov
        # trailing revenue growth from the statements when the overview has none
        _qrg = reconciled_revenue_yoy(all_data)
        if _qrg is None:
            _qrg = safe_float(_ovs.get("QuarterlyRevenueGrowthYOY"))
        if _qrg is None and len(_iq_all) >= 5:
            _r0, _r4 = safe_float(_iq_all[0].get("totalRevenue")), safe_float(_iq_all[4].get("totalRevenue"))
            if _r0 and _r4: _qrg = _r0 / _r4 - 1.0
        if fair_val_result and fair_val_result.get("lifecycle_stage"):
            _CRUCIBLE_STORY_STAGE = fair_val_result.get("lifecycle_stage")
        else:
            # stage from trailing facts only (no ML forecast): same classifier, forward=trailing
            _pf_min = {"rev_growth": (_qrg if _qrg is not None else 0.0), "eps_growth": 0.0}
            try:
                _CRUCIBLE_STORY_STAGE = _lifecycle_stage(_pf_min, _ovs, False, consensus=all_data.get("earnings_estimates_consensus"),
                                                         trailing_rev_growth=reconciled_revenue_yoy(all_data))
            except Exception:  # noqa: BLE001
                _CRUCIBLE_STORY_STAGE = "young-growth"
        # GAAP distortion is measured below (no model judges or overrides the stage).
        _stage_check = None
        try:
            import damodaran_portfolio as _dpv

            try:
                # Event RECALL needs the strongest model first (flash-lite returned an empty
                # list for Apple's own CEO succession); Claude leads, GPT and Gemini back up.
                def _events_call(system, user):
                    out = _dpv._call_claude_json(system, user)
                    if not (out or {}).get("events"):
                        out2 = _dpv._call_frontier_json(user, system=system)
                        if (out2 or {}).get("events"):
                            return out2
                    if out is None:
                        out = _dpv._call_gemini_json(system, user)
                    return out
                all_data["corp_events"] = _corporate_events_brief(
                    ticker, _ovs.get("Name"), _events_call)
                all_data["corp_actions"] = _corporate_actions_brief(ticker, _ovs.get("Name"), _events_call,
                                                                    headlines=[(a.get("title") or "")[:120] for a in (all_data.get("news") or [])[:25]])
                _stq0 = (((all_data.get("income") or {}).get("quarterlyReports") or [{}])[0] or {}).get("fiscalDateEnding")
                _rst_early = all_data.get("restatement")
                all_data["restatement"] = _restatement_from_actions(all_data["corp_actions"], _stq0)
                if not all_data["restatement"]:
                    # Data-driven fallback: model recall lags reality (it still had S&P Global's
                    # July-2026 Mobility spin as "pending"). If sell-side consensus revenue for the
                    # CURRENT fiscal year sits well below reported TTM revenue while the company is
                    # growing, analysts are modelling a smaller continuing company — a completed
                    # spin-off/divestiture is the only ordinary explanation.
                    all_data["restatement"] = _restatement_from_consensus(
                        all_data.get("earnings_estimates_consensus") or {}, safe_float(_ovs.get("RevenueTTM")),
                        safe_float(_ovs.get("QuarterlyRevenueGrowthYOY")), all_data["corp_actions"], _stq0,
                        edgar=all_data.get("edgar_actions"))
                # Canonical leadership dates (8-K Item 5.02) and the announcement date of the
                # divested unit (earliest 8-K mention) — used to correct recalled event dates.
                all_data["edgar_officers"] = _edgar_officer_changes(ticker)
                all_data["form4"] = _edgar_form4(ticker)
                all_data["insider_canon"] = _insider_canon(all_data["form4"], all_data.get("insider"))
                # The provider feed records only acquired/disposed, so a scheduled plan sale, a
                # tax-withholding disposition and a discretionary sale all scored alike. Form 4
                # carries the transaction class, so its type-weighted factor REPLACES the untyped
                # score everywhere the personas and the catalyst gate read it.
                _icn = all_data["insider_canon"]; _ifac = _icn.get("factor") or {}
                if sent is not None and _icn.get("n_transactions"):
                    _comp = sent.setdefault("components", {})
                    _prev = _comp.get("insider_score", sent.get("insider_score"))
                    for _d in (sent, _comp):
                        _d["insider_score_untyped"] = _prev
                        _d["insider_score"] = _ifac.get("score")
                        _d["insider_detail"] = _icn.get("characterization") or _d.get("insider_detail")
                        _d["insider_score_basis"] = _ifac.get("basis")
                    if _prev is not None and _ifac.get("score") is None:
                        print(f"  [insider] untyped score {_prev:+.2f} withdrawn — every sale in the "
                              f"window was non-discretionary ({_icn.get('characterization')})")
                    elif _prev is not None and _ifac.get("score") is not None and abs(_prev - _ifac["score"]) > 0.05:
                        print(f"  [insider] score {_prev:+.2f} → {_ifac['score']:+.2f} after weighting by transaction class")
                if all_data["insider_canon"].get("note"):
                    print(f"  [form4] {all_data['insider_canon']['note'][:140]}")
                _ceo_eff = None
                _today_s = datetime.now().strftime("%Y-%m-%d")
                _cands = sorted({d for _o in all_data["edgar_officers"] if _o.get("ceo_related") for d in (_o.get("effective_dates") or []) if d <= _today_s})
                if _cands:
                    _ceo_eff = _cands[-1]            # most recent completed CEO transition
                if _ceo_eff:
                    print(f"  [edgar] CEO transition effective {_ceo_eff} (8-K Item 5.02)")
                    for _ev in all_data.get("corp_events") or []:
                        if re.search(r"\bCEO\b|chief executive", str(_ev.get("event") or ""), re.I) and str(_ev.get("date") or "")[:7] != _ceo_eff[:7]:
                            _ev["event"] = str(_ev["event"]) + f" [date per 8-K Item 5.02: effective {_ceo_eff}]"; _ev["date"] = _ceo_eff[:7]
                _rstb = (all_data.get("restatement") or {}).get("business")
                if _rstb and _rstb != "a divested business":
                    _key0 = next((t.strip("(),.") for t in _rstb.split() if len(t.strip("(),.")) > 3), None)
                    _eff0 = (all_data["restatement"].get("effective") or datetime.now().strftime("%Y-%m-%d"))[:10]
                    try:
                        _since0 = (datetime.strptime(_eff0, "%Y-%m-%d") - timedelta(days=600)).strftime("%Y-%m-%d")
                    except ValueError:
                        _since0 = (datetime.now() - timedelta(days=600)).strftime("%Y-%m-%d")
                    _ann = _edgar_first_8k_mention(ticker, _key0, ("separation", "spin-off", "spinoff", "divestiture"), _since0)
                    if _ann:
                        all_data["restatement"]["announced"] = _ann
                        print(f"  [edgar] {_key0} separation first appears in an 8-K on {_ann}")
                        for _ev in all_data.get("corp_events") or []:
                            if _key0.lower() in str(_ev.get("event") or "").lower() and re.search(r"spin|separat|divest", str(_ev.get("event") or ""), re.I) and str(_ev.get("date") or "")[:7] != _ann[:7]:
                                _ev["event"] = str(_ev["event"]) + f" [announcement date per 8-K: {_ann}]"; _ev["date"] = _ann[:7]
                # Recalled "pending" deals: ask the filings. Full-text search of the company's own
                # 8-K/10-Q/10-K text for a completion sentence naming the unit.
                for _a in all_data["corp_actions"]:
                    # pending → resolve status; completed-by-recall → verify and take the filing's date
                    # (recall dated the OSTTRA close "2025-05"; the 10-Q says 2025-10-10)
                    if _a.get("status") == "pending" or (_a.get("status") == "completed" and _a.get("kind") != "acquisition" and not _a.get("evidence")):
                        _ds = _edgar_deal_status(ticker, _a.get("business"), (datetime.now() - timedelta(days=550)).strftime("%Y-%m-%d"))
                        if _ds:
                            _a["status"] = "completed"; _a["effective"] = _ds["date"] or _a.get("effective")
                            _a["evidence"] = {"form": _ds["form"], "date": _ds["date"], "filed": _ds.get("filed"),
                                              "snippet": _ds["snippet"], "url": _ds["url"]}
                            _a["note"] = (_a.get("note") or "") + f" [closed per {_ds['form']} filed {_ds['date']}: \"{_ds['snippet'][:120]}\"]"
                            print(f"  [edgar] {_a['business']}: completed per {_ds['form']} {_ds['date']} — {_ds['snippet'][:100]}")
                # A recalled "pending" spin-off/divestiture followed by an 8-K Item 2.01 filing is complete.
                for _a in all_data["corp_actions"]:
                    if _a.get("kind") in ("spin_off", "divestiture") and _a.get("status") == "pending":
                        _aw = {w.lower().strip("(),.") for w in str(_a.get("business") or "").split() if len(w) > 3}
                        for _e in all_data.get("edgar_actions") or []:
                            _ew = {w.lower().strip("(),.") for w in str(_e.get("excerpt") or "").split() if len(w) > 3}
                            # only when the filing text names this unit — an 8-K for one deal must
                            # not "complete" a different one (OSTTRA was stamped with Mobility's 8-K)
                            if _e.get("kind") == "completed_acquisition_or_disposition" and (_aw & _ew):
                                _a["status"] = "completed"; _a["effective"] = _a.get("effective") or _e["filed"]
                                _a["note"] = (_a.get("note") or "") + f" [8-K Item 2.01 filed {_e['filed']}]"; break
                # When the consensus signal says the divestiture has happened, the recalled action's
                # "pending" status is stale: mark it completed (inferred) so no analyst calls it
                # pending while the caveat says it closed (SPGI PM: "both separations still pending").
                _rst_biz = (all_data.get("restatement") or {}).get("business")
                if _rst_biz and _rst_biz != "a divested business":
                    for _a in all_data["corp_actions"]:
                        if _a.get("business") == _rst_biz and _a.get("status") == "pending":
                            _a["status"] = "completed"; _a["note"] = (_a.get("note") or "") + " [completion inferred from consensus revenue]"
                all_data["pending_deals"] = [a for a in all_data["corp_actions"] if a.get("status") == "pending"]
                if all_data.get("corp_actions"):
                    print("  [actions] " + " | ".join(f"{a['kind']} {a['business']} ({a['status']}{', ' + a['effective'] if a.get('effective') else ''})"
                                                     for a in all_data["corp_actions"]))
                if all_data.get("restatement") and (all_data["restatement"] is not _rst_early):
                    print(f"  {YELLOW}[restatement] {all_data['restatement']['note']}{RESET}")
                # business identity comes from the canonical state only: the filed segments of the valuation perimeter, a
                # finer decomposition the company itself files, or none. A company that files one segment is valued as one
                # business; nothing is reconstructed from model memory.
                all_data["business_decomposition"] = _business_decomposition(ticker, all_data)
                all_data["segment_brief"] = _filed_segment_brief(all_data) or _product_line_brief(all_data["business_decomposition"])
                _bd0 = all_data["business_decomposition"]
                print(f"  [businesses] {_bd0['mode']} from {_bd0['source_type']}: "
                      + ("; ".join(f"{b['business_id']} = {b['name']}" + ("" if b["story_eligible"] else " (not eligible: left the company)")
                                   for b in _bd0["businesses"]) or "no decomposition"))
                if all_data.get("segment_brief"):
                    _sb = all_data["segment_brief"]
                    print("  [segments] " + "; ".join(f"{g['name']} {g['revenue_share']*100:.0f}%"
                          + (f" @{g['margin']*100:.0f}% {g['margin_basis'] or ''}" if g.get('margin') is not None else "")
                          for g in _sb["segments"][:4])
                          + ("; absorbed: " + ", ".join(a["business"] for a in _sb["absorbed"]) if _sb.get("absorbed") else ""))
                try:
                    import canonical as _cn2
                    if all_data.get("issuer") is not None:
                        _prev_iss = dict(all_data["issuer"])
                        all_data["issuer"] = _cn2.classify_issuer(ticker, _ovs, {"cik": _EDGAR_CIK.get(ticker.upper()), "sub": _EDGAR_SUB.get(_EDGAR_CIK.get(ticker.upper())) or {}},
                                                                  all_data.get("xbrl"), segment_brief=all_data.get("segment_brief"))
                        # facts already resolved from primary sources are not re-derived away
                        for _k3 in ("is_adr", "adr_ratio", "adr_source"):
                            if _prev_iss.get(_k3) is not None:
                                all_data["issuer"][_k3] = _prev_iss[_k3]
                        all_data["valuation_framework"] = _cn2.select_valuation_framework(all_data["issuer"], _ovs, all_data.get("segment_brief"))
                        if all_data["valuation_framework"].get("sotp_required"):
                            print(f"  [framework] SUM-OF-PARTS REQUIRED — {all_data['valuation_framework']['valuation_framework_reason'][:120]}")
                        import policy as _pol2
                        all_data["policy"] = _pol2.ApplicabilityPolicy.for_issuer(all_data["issuer"])
                except Exception:  # noqa: BLE001
                    pass
                all_data["leadership"] = _wiki_leadership(_ovs.get("Name") or ticker)
                if all_data.get("leadership"):
                    print(f"  [leadership] {all_data['leadership'][:100]}")
                all_data["leadership_canon"] = _leadership_canon(
                    all_data.get("edgar_officers"), all_data.get("leadership"), all_data.get("form4"),
                    issuer=all_data.get("issuer"), officers_20f=all_data.get("officers_20f"))
                # primary-source resolution: if the 6-month Form 4 window did not yield a firm-level CEO/CFO,
                # look back 18 months (officer titles are self-reported to the SEC on every Form 4)
                if not all((all_data["leadership_canon"].get(t) or {}).get("source", "").startswith("SEC") for t in ("CEO", "CFO")):
                    try:
                        _f4_long = _edgar_form4(ticker, months=18, max_filings=60)
                        if _f4_long:
                            _lc_long = _leadership_canon(all_data.get("edgar_officers"), all_data.get("leadership"), _f4_long,
                                                         issuer=all_data.get("issuer"), officers_20f=all_data.get("officers_20f"))
                            for _t in ("CEO", "CFO", "COO"):
                                if (_lc_long.get(_t) or {}).get("source", "").startswith("SEC"):
                                    all_data["leadership_canon"][_t] = _lc_long[_t]
                    except Exception:  # noqa: BLE001
                        pass
                _lcl = " · ".join(f"{k} {v['name']} ({v['source'][:34]})" for k, v in all_data["leadership_canon"].items() if isinstance(v, dict) and v.get("name"))
                if _lcl: print(f"  [leadership] canonical: {_lcl}")
                # §21 — a verdict per role. "CEO resolved, CFO missing" is not a pass.
                try:
                    import leadership as _ldm
                    _lres = {k: v for k, v in all_data["leadership_canon"].items() if isinstance(v, dict)}
                    all_data["leadership_status"] = _ldm.role_statuses(_lres, roles_required=("CEO", "CFO"),
                                                                       roles_used=("CEO", "CFO"))
                    all_data["leadership_completeness"] = _ldm.completeness(all_data["leadership_status"],
                                                                           roles_required=("CEO", "CFO"),
                                                                           roles_used=("CEO", "CFO"))
                    print(f"  [leadership] regime {all_data['leadership_canon'].get('_regime')} — "
                          f"{all_data['leadership_completeness']['detail']}")
                except Exception as _le:  # noqa: BLE001
                    print(f"  [leadership] per-role status skipped: {str(_le)[:60]}")
                # Peer comparables (panel) + catalyst calendar
                _re_svc = (str(_ovs.get("Sector") or "").upper() == "REAL ESTATE" and not _is_reit_like(_ovs))
                _nom = _nominate_peers(ticker, _ovs.get("Name"), _ovs.get("Industry"), _ovs.get("Description"), _events_call,
                                       hint=("This is a real-estate SERVICES company (brokerage / property & facilities management / advisory), NOT a REIT: nominate service and brokerage peers, never REITs." if _re_svc else None))
                all_data["peer_comps"] = _peer_comps(ticker, _ovs.get("Sector"), safe_float(_ovs.get("MarketCapitalization")), prefer=_nom, exclude_reits=_re_svc)
                if all_data.get("peer_comps"):
                    _pp = all_data["peer_comps"]
                    print(f"  [peers] {len(_pp['peers'])} {_pp['sector']} peers · subject percentiles {_pp['percentiles']}")
                _cats = []
                try:
                    for e in (all_data.get("earnings_calendar_12m") or [])[:2]:
                        _dte = e.get("reportDate") or e.get("report_date") or e.get("date")
                        if _dte: _cats.append({"date": str(_dte)[:10], "event": "Next earnings report", "kind": "earnings"})
                    if _ovs.get("ExDividendDate") and str(_ovs.get("ExDividendDate")) not in ("None", ""):
                        _cats.append({"date": str(_ovs.get("ExDividendDate"))[:10], "event": f"Ex-dividend (${_ovs.get('DividendPerShare')}/yr)", "kind": "dividend"})
                    for e in (all_data.get("corp_events") or [])[:5]:
                        _cats.append({"date": e.get("date"), "event": e.get("event"), "kind": "corporate (recalled)"})
                except Exception:
                    pass
                _today = datetime.now().strftime("%Y-%m-%d")
                all_data["catalysts"] = [c for c in _cats if c.get("kind") == "corporate (recalled)" or (c.get("date") and str(c["date"]) >= _today)]
                all_data["risk_card"] = _risk_card(all_data, _ovs)
                if all_data["corp_events"]:
                    print("  [events] " + " | ".join(f"{e['date']}: {e['event'][:60]}" for e in all_data["corp_events"][:3]))
            except Exception:  # noqa: BLE001
                pass
            # GAAP distortion is MEASURED, not judged by a model: reported EPS far from adjusted EPS on the
            # reconciled figures means statement-earnings multiples misstate the business, so those
            # methods are down-weighted and the composite re-blended — recorded as a composite step
            _stage_check = _gaap_distortion_check(all_data, _CRUCIBLE_STORY_STAGE)
            if _stage_check["gaap_distortion"] and fair_val_result and fair_val_result.get("methods"):
                _gaap_reblend(fair_val_result, _stage_check, current_price)
            if fair_val_result is not None:
                fair_val_result["stage_check"] = _stage_check
        except Exception as _ve:  # noqa: BLE001
            if args.verbose: print(f"  [stage-check] unavailable: {str(_ve)[:60]}")

        _fin = (str(_ovs.get("Sector") or "").upper() == "FINANCIAL SERVICES" and
                any(k in str(_ovs.get("Industry") or "").upper() for k in ("BANK", "INSURANCE", "CAPITAL MARKETS", "ASSET MANAGEMENT", "CREDIT")))
        _reit = _is_reit_like(_ovs)
        # Financials, banks AND REITs skip the revenue×margin×sales-to-capital FCFF story:
        # a balance-sheet business (banks: FCFE/book×ROE; REITs: AFFO) isn't a revenue-driven
        # FCFF firm, so the mechanical valuation (Method 6 book-value / AFFO path) carries value.
        # Damodaran values EVERY operating company with a DCF across the whole life cycle: the stage
        # shapes the story (growth, reinvestment, margins, failure risk), it does not decide whether
        # an intrinsic valuation exists. Balance-sheet businesses use their own engines (policy).
        _story_eligible = True
        _fw_story_ok = (all_data["policy"].story_eligible if all_data.get("policy") is not None else (all_data.get("valuation_framework") or {}).get("story_eligible", True))
        if not _fw_story_ok:
            print(f"  [policy] story valuation not applicable for a {(all_data.get('policy').issuer_class if all_data.get('policy') is not None else '?')}: {(all_data.get('policy').valuation_engine if all_data.get('policy') is not None else '')}")
        if (_story_eligible and _fw_story_ok and not _fin and not _reit and _rev_ov and _shares_ov):
            import story_dcf as _sd
            import damodaran_portfolio as _dp
            _ov = all_data.get("overview") or {}
            _bq = ((all_data.get("balance") or {}).get("quarterlyReports") or [{}])[0] or {}
            _dm = (fair_val_result or {}).get("dcf_meta") or {}
            _di = all_data.get("damodaran_inputs") or {}
            _b0, _bst = _sd.damodaran_betas(safe_float(_ov.get("Beta")), _ov.get("Sector"), _ov.get("Industry"), bottom_up=_di.get("beta"))
            # interest coverage (TTM EBIT / TTM interest) → synthetic rating → cost of debt
            _iq = ((all_data.get("income") or {}).get("quarterlyReports") or [])[:4]
            _ebit_ttm = sum((safe_float(q.get("ebit")) or safe_float(q.get("operatingIncome")) or 0.0) for q in _iq)
            _int_ttm  = sum(abs(safe_float(q.get("interestExpense")) or 0.0) for q in _iq)
            _cov = (_ebit_ttm / _int_ttm) if _int_ttm > 0 else None
            _news = "; ".join((a.get("title") or "")[:90] for a in (all_data.get("news") or [])[:8])
            # Alpha Vantage balance-sheet debt fields vary by filer; fall through several
            # (Realty Income came out at 4% debt weight against a real ~35% because the first
            # field was empty and the fallback caught only short-term debt).
            _lt = (safe_float(_bq.get("longTermDebt")) or safe_float(_bq.get("longTermDebtNoncurrent")) or 0.0)
            _stv = (safe_float(_bq.get("shortTermDebt")) or safe_float(_bq.get("currentDebt")) or
                    safe_float(_bq.get("currentLongTermDebt")) or 0.0)
            _debt = safe_float(_bq.get("shortLongTermDebtTotal")) or (_lt + _stv)
            _eec = all_data.get("earnings_estimates_consensus") or {}
            # Continuing-company revenue base when a spin-off/divestiture completed inside the TTM
            # window: prefer the sell-side current-FY consensus (analysts model the remaining
            # business), else strip the recalled revenue share; never value a divested unit.
            # the story's starting point is the typed StoryBase (story_base.py): one business, one period, one
            # perimeter adjustment shared with every other module. No module re-estimates the separated business.
            _sbase = all_data.get("story_base") or {}
            _rev_story = _sbase.get("revenue") or _rev_ov
            _m_story = _sbase.get("operating_margin") if _sbase.get("operating_margin") is not None else _opm_ov
            _rev_note = f"{_sbase.get('revenue_label')}: {_sbase.get('derivation')}" if _sbase.get("perimeter") != "consolidated" else None
            _m_note = f"{_sbase.get('label')} ({_sbase.get('quality_status')})" if _sbase.get("perimeter") != "consolidated" else None
            _rst = all_data.get("restatement")
            if _rev_note:
                print(f"  [story] {_rev_note}")
            _ctx = dict(ticker=ticker, name=_ov.get("Name"), sector=_ov.get("Sector"), industry=_ov.get("Industry"),
                        description=_ov.get("Description"), revenue=_rev_story, margin_base_note=_m_note,
                        revenue_reported=_rev_ov, revenue_base_note=_rev_note,
                        restatement=all_data.get("restatement"), pending_deals=all_data.get("pending_deals"),
                        rev_growth=_qrg,
                        op_margin=_m_story, net_margin=safe_float(_ov.get("ProfitMargin")),
                        market_cap=_mcap_ov, price=current_price,
                        shares=_shares_ov, debt=_debt,
                        cash=safe_float(_bq.get("cashAndCashEquivalentsAtCarryingValue")) or 0.0,
                        book_equity=safe_float(_bq.get("totalShareholderEquity")),
                        damodaran=_di,
                        # one source for each: the validated 10-year Treasury and his ERP (+ CRP) — never
                        # the mechanical DCF's copy, which does not exist when the statistical forecast fails
                        rf=canonical_risk_free(all_data), erp=_di.get("erp") or _dm.get("erp") or DAMODARAN_ERP,
                        beta=_b0, beta_stable=_bst, interest_coverage=_cov, is_reit=_reit,
                        stable_cost_of_capital=_di.get("stable_cost_of_capital"),
                        stable_cost_of_capital_basis=_di.get("stable_cost_of_capital_basis"),
                        stable_market_beta=_di.get("stable_market_beta"),
                        # the benchmark is rebuilt inside the valuation at each case's rf; the beta passes a provenance
                        # gate before it may shape the stable rate
                        benchmark_region=_di.get("region"),
                        beta_quality=__import__("damodaran_data").beta_quality(_di, bool((all_data.get("issuer") or {}).get("is_financial"))),
                        beta_source=_di.get("beta_basis"), beta_industry=_di.get("industry"),
                        stage=_CRUCIBLE_STORY_STAGE, news=_news,
                        cons_rev_cur=_eec.get("current_year_revenue"), cons_rev_next=_eec.get("next_year_revenue"),
                        cons_fy_cur=_eec.get("current_year_date"), cons_fy_next=_eec.get("next_year_date"),
                        cons_n=_eec.get("analyst_count"),
                        segment_brief=all_data.get("segment_brief"),
                        business_decomposition=(all_data.get("business_decomposition")
                                                or all_data.setdefault("business_decomposition", _business_decomposition(ticker, all_data))),
                        gaap_distortion_note=((_stage_check or {}).get("distortion_reason")
                                              if (_stage_check or {}).get("gaap_distortion") else None))
            # Two INDEPENDENT storytellers, led by different models, so one model's habits
            # don't set the consensus (GPT-first and Claude-first; Gemini as the shared fallback).
            # Story JSON (with segments) is long; reasoning models spend the completion budget
            # thinking first, so give each storyteller a wide budget and log why a seat failed.
            def _why(tag):
                _e = _dp.last_error()
                if _e: print(f"    [story] {tag}: {_e}")
            def _story_call(system, user):
                out = _dp._call_frontier_json(user, system=system, max_tokens=4000)
                if out is None: _why("storyteller 1 (GPT)"); out = _dp._call_claude_json(system, user, temperature=0.3, max_tokens=4500)
                if out is None: _why("Claude fallback"); out = _dp._call_gemini_json(system, user, max_tokens=6000)
                if out is None: _why("Gemini fallback")
                return out
            def _story_call2(system, user):
                out = _dp._call_claude_json(system, user, temperature=0.3, max_tokens=4500)
                if out is None: _why("storyteller 2 (Claude)"); out = _dp._call_gemini_json(system, user, max_tokens=6000)
                if out is None: _why("Gemini fallback"); out = _dp._call_frontier_json(user, system=system, max_tokens=4000)
                if out is None: _why("GPT fallback")
                return out
            # Third, Gemini-led storyteller: the consensus is the MEDIAN of three sets of
            # assumptions, so a single drifting story can't move the value (run-to-run stability).
            def _story_call3(system, user):
                out = _dp._call_gemini_json(system, user, max_tokens=6000)
                if out is None: _why("storyteller 3 (Gemini)"); out = _dp._call_claude_json(system, user, temperature=0.3, max_tokens=4500)
                if out is None: _why("Claude fallback")
                return out
            # progress marker for the web page: the framework's INPUTS for this company (the same values the report
            # shows later), never a value, rating or storyteller view. analyst.py forwards whitelisted fields only.
            _sbp = all_data.get("segment_brief") or {}
            _facts = {"industry": _di.get("industry"), "country": _di.get("country"), "beta": _di.get("beta"),
                      "beta_basis": "business mix" if _di.get("business_mix") else "industry",
                      "erp": _di.get("erp"), "erp_month": _di.get("erp_month"), "crp": _di.get("crp"),
                      "stage": _CRUCIBLE_STORY_STAGE,
                      "segments": [g.get("name") for g in (_sbp.get("segments") or [])] if _sbp.get("filed") else [],
                      "segment_build": bool(_sbp.get("segment_build_required")),
                      "consensus": bool(_eec.get("current_year_revenue")),
                      "ml_rank": (_panel_state(ticker)[0] or {}).get("state")}
            print("@@progress story-start " + json.dumps(_facts, default=str), flush=True)
            story_val_result = _sd.story_value(_ctx, _story_call, _story_call2, _story_call3, verbose=args.verbose)
            if not story_val_result:
                print(f"  Damodaran story-FCFF: n/a — {getattr(_sd, 'LAST_REASON', None) or 'unknown reason'}")
            if story_val_result:
                story_val_result["stage"] = _CRUCIBLE_STORY_STAGE
                # Sensitivity grid on the consensus assumptions (deterministic re-runs of the
                # same FCFF model) — value is a range, not a point.
                try:
                    _A0 = story_val_result["assumptions"]
                    _base = {"tam_at_maturity_usd_b": _A0["tam_usd_b"], "plausible_peak_share": _A0["peak_share"],
                             "years_to_maturity": _A0["years"], "target_operating_margin": _A0["margin_target"],
                             "sales_to_capital": _A0["sales_to_capital"], "failure_probability": _A0["failure_probability"],
                             "story": story_val_result.get("story"), "key_risk": story_val_result.get("key_risk")}
                    def _run(ctx_over=None, **a_over):
                        _c = dict(_ctx); _c.update(ctx_over or {})
                        _a = dict(_base); _a.update(a_over)
                        # strict=False: the base already carries the post-guardrail margin, so
                        # the perturbation must be allowed to move it (a company at its sector
                        # cap otherwise shows "margin +5 pts -> +0.0%").
                        _r = _sd.story_fcff(_c, _a, strict=False)
                        _run.last_path = ((_r or {}).get("assumptions") or {}).get("discount_rate_path")
                        return (_r["value"] if _r else None)
                    _v0 = story_val_result["value"]
                    _sens = []
                    # each case is structured (driver, shift, shift kind) so the renderer shows the shift
                    # as a typed figure; "case" keeps a readable label for logs only
                    for lbl, drv, shift, kind, kw, co in (
                        ("Discount rate +1 pt", "Risk-free rate", 0.01, "PP", {}, {"rf": _ctx["rf"] + 0.01}),
                        ("Discount rate −1 pt", "Risk-free rate", -0.01, "PP", {}, {"rf": max(0.01, _ctx["rf"] - 0.01)}),
                        ("Target margin +5 pts", "Maturity operating margin", 0.05, "PP", {"target_operating_margin": _A0["margin_target"] + 0.05}, None),
                        ("Target margin −5 pts", "Maturity operating margin", -0.05, "PP", {"target_operating_margin": _A0["margin_target"] - 0.05}, None),
                        ("Peak share +25%", "Peak market share", 0.25, "REL", {"plausible_peak_share": _A0["peak_share"] * 1.25}, None),
                        ("Peak share −25%", "Peak market share", -0.25, "REL", {"plausible_peak_share": _A0["peak_share"] * 0.75}, None),
                        ("Reinvestment 20% cheaper", "Sales to capital", 0.25, "REL", {"sales_to_capital": _A0["sales_to_capital"] * 1.25}, None),
                        ("Reinvestment 20% dearer", "Sales to capital", -0.20, "REL", {"sales_to_capital": _A0["sales_to_capital"] * 0.8}, None),
                    ):
                        _v = _run(co, **kw)
                        _dp = getattr(_run, "last_path", None) or {}
                        if _v: _sens.append({"case": lbl, "driver": drv, "shift": shift, "shift_kind": kind,
                                             "value": round(_v, 2), "vs_base": round(_v / _v0 - 1.0, 3),
                                             "implied": round(_v / current_price - 1.0, 3),
                                             # the whole rate path of this case (rebuilt, not the base's)
                                             "discount_rate_path": {k: _dp.get(k) for k in ("rf", "initial_wacc", "terminal_wacc",
                                                                                            "mature_benchmark_wacc", "yearly_rates",
                                                                                            "terminal_spread")}})
                    story_val_result["sensitivity"] = _sens
                    # story vs mechanical bridge (same FCFF model re-run one assumption at a time) and dispersion
                    _mechv = (((fair_val_result or {}).get("methods") or {}).get("DCF") or {}).get("price")
                    story_val_result["bridge"] = _sd.story_bridge(_ctx, _A0, story_val_result["value"], _mechv, (fair_val_result or {}).get("dcf_meta"))
                    _pvs = story_val_result.get("persona_values") or []
                    if len(_pvs) >= 2:
                        _lo, _hi = min(_pvs), max(_pvs); _spr = (_hi - _lo) / max(_lo, 1e-9)
                        story_val_result["dispersion"] = {"min": _lo, "max": _hi, "spread": _spr, "flag": _spr > 0.5,
                                                          "note": ("storytellers disagree by more than 50% — the point estimate is a low-confidence median; treat the value as the range" if _spr > 0.5 else "storyteller dispersion within 50%")}
                    _br = story_val_result.get("bridge") or {}
                    if _br.get("gap") is not None:
                        print(f"  [bridge] story vs mechanical {_br['gap']*100:+.0f}% ({_br.get('level')}); {_br.get('note')}")
                except Exception as _sx:  # noqa: BLE001
                    if args.verbose: print(f"  sensitivity failed: {str(_sx)[:60]}")
                # Reverse-engineer the price (his "what you must believe" check) on the same
                # consensus assumptions: revenue needed at the story margin; margin needed at
                # the story revenue.
                try:
                    _A0 = story_val_result["assumptions"]
                    _beA = {"tam_at_maturity_usd_b": _A0["tam_usd_b"], "plausible_peak_share": _A0["peak_share"],
                            "years_to_maturity": _A0["years"], "target_operating_margin": _A0["margin_target"],
                            "sales_to_capital": _A0["sales_to_capital"], "failure_probability": _A0["failure_probability"]}
                    _be = _sd.story_breakeven(_ctx, _beA)
                    if _be:
                        story_val_result["breakeven"] = _be
                        if _be.get("revenue_needed_N_usd_b"):
                            print(f"    breakeven: price needs ${_be['revenue_needed_N_usd_b']:,.0f}B revenue in year {_be['years']} "
                                  f"({_be['rev_cagr_needed']*100:.1f}%/yr) vs story ${_be['story_revenue_N_usd_b']:,.0f}B; "
                                  + (f"or a {_be['margin_needed']*100:.0f}% margin vs story {_be['story_margin']*100:.0f}%" if _be.get('margin_needed') is not None
                                     else f"no margin below 70% gets there (story {_be['story_margin']*100:.0f}%)"))
                        else:
                            print(f"    breakeven: {_be.get('revenue_needed_note') or _be.get('margin_needed_note') or 'n/a'}")
                except Exception as _bx:  # noqa: BLE001
                    if args.verbose: print(f"  breakeven failed: {str(_bx)[:60]}")
                if fair_val_result is not None:
                    fair_val_result["story"] = story_val_result
                _a = story_val_result["assumptions"]
                print(f"  Damodaran story-FCFF: ${story_val_result['value']:.2f} ({story_val_result['implied']*100:+.1f}%) — "
                      f"TAM ${_a['tam_usd_b']:,.0f}B × {_a['peak_share']*100:.0f}% share → ${_a['revenue_N_usd_b']:,.0f}B rev in {_a['years']}y, "
                      f"margin {_a['margin_now']*100:.0f}%→{_a['margin_target']*100:.0f}%, WACC {_a['wacc_initial']*100:.1f}%→{_a['wacc_terminal']*100:.1f}%")
                print(f"    story: {story_val_result['story'][:140]}")
                if story_val_result.get("persona_n", 1) == 2:
                    _pv = story_val_result.get("persona_values") or []
                    print(f"    consensus of 2 personas — individual values ${_pv[0]:,.2f} / ${_pv[1]:,.2f}")
    except Exception as _se:  # noqa: BLE001
        if args.verbose: print(f"  story-FCFF unavailable: {str(_se)[:70]}")
    if pred_fund_result:
        metrics = pred_fund_result.get("metrics_predicted", [])
        print(f"{GREEN}{len(metrics)} metrics ({', '.join(metrics[:4])}...){RESET}")
        if fair_val_result:
            fvc = GREEN if fair_val_result["expected_return"] > 0 else RED
            methods_str = ", ".join(fair_val_result["methods"].keys())
            print(f"  Fair value methods: {methods_str}")
            print(f"  Blended fair value: {fvc}${fair_val_result['fair_price']:.2f}{RESET}  "
                  f"implied: {fvc}{fair_val_result['expected_return']*100:+.1f}%{RESET}  "
                  f"(conf:{fair_val_result['confidence']*100:.0f}%)  "
                  f"{CYAN}[lifecycle: {fair_val_result.get('lifecycle_stage','?')} — stage-weighted]{RESET}")
            if pred_fund_result.get("eps_annual"):
                eg = pred_fund_result.get("eps_growth", 0)
                gc = GREEN if eg > 0 else RED if eg < 0 else YELLOW
                ec = pred_fund_result.get("eps_confidence", 0)
                print(
                    f"  Predicted EPS: ${pred_fund_result['eps_annual']:.2f}  "
                    f"(growth: {gc}{eg*100:+.1f}%{RESET}  conf:{ec*100:.0f}%)"
                )
                eaf = pred_fund_result.get("eps_anchor_flag")
                if eaf:
                    print(f"    {DIM}EPS anchor: {eaf}{RESET}")
        else:
            print(f"  {YELLOW}Fair value derivation unavailable (missing multiples){RESET}")
    else:
        print(f"{DIM}insufficient quarterly data{RESET}")
        pred_fund_result = None
        fair_val_result  = None

    print(f"  Macro analysis (yearly + yield curve)...", end=" ", flush=True)
    macro_result = analyze_macro_yearly(all_data)
    if macro_result:
        mc = GREEN if macro_result["expected_return"] > 0 else RED
        yc = macro_result["signals"].get("yield_curve_signal", "?")
        print(f"{mc}{macro_result['expected_return']*100:+.2f}%{RESET} (conf:{macro_result['confidence']*100:.0f}% YC:{yc})")
    else:
        print(f"{DIM}unavailable{RESET}")

    print(f"  ML ensemble (fund+macro features)...", end=" ", flush=True)
    ml_result, _, _ = train_ml_model_yearly(daily, spy_df, pit_fund, pit_macro, extra=ml_extra)
    if ml_result:
        mlc = GREEN if ml_result["expected_return"] > 0 else RED
        # When the model can't predict direction (dir_acc ≤ ~coin flip, negative RankIC,
        # or huge MAE) its confidence is floored, so the blend already all-but-ignores it.
        # Say so on the line — otherwise a wild point estimate looks like a real signal.
        _da = float(ml_result.get('backtest_dir_accuracy', 0.5) or 0.5)
        _ric = float(ml_result.get('backtest_rank_ic', 0) or 0)
        _mae = float(ml_result.get('backtest_mae', 0) or 0)
        _unreliable = _da < 0.45 or _ric < 0 or _mae > 0.60
        _flag = f" {YELLOW}⚠ low reliability → near-zero weight in blend{RESET}" if _unreliable else ""
        print(f"{mlc}{ml_result['expected_return']*100:+.2f}%{RESET} "
              f"(dir_acc:{ml_result['backtest_dir_accuracy']*100:.0f}% "
              f"RankIC:{ml_result.get('backtest_rank_ic', 0):.3f} "
              f"OOS:{ml_result.get('n_oos_samples', '?')} "
              f"MAE:{ml_result['backtest_mae']*100:.1f}% "
              f"feat:{ml_result['n_features']}){_flag}")
    else:
        print(f"{DIM}insufficient data{RESET}")

    print(f"  Cross-sectional screen (composite + 12M earnings model + lifecycle DCF)...", flush=True)
    _t10q = all_data.get("treasury_10y") or []
    _di_q = all_data.get("damodaran_inputs") or {}
    quant_ctx = scanner_quant_context(ticker, current_price, use_llm_dcf=True, verbose=args.verbose,
                                      fx_usd=float(all_data.get("fx_usd") or 1.0),
                                      market={"rf": (safe_float(_t10q[0].get("value")) / 100.0 if _t10q and safe_float(_t10q[0].get("value")) else None),
                                              "beta": _di_q.get("beta"), "erp": _di_q.get("erp")})
    if quant_ctx:
        if quant_ctx.get("comp_rank"):
            st = quant_ctx.get("styles", {})
            print(f"    composite screen: rank #{quant_ctx['comp_rank']}/{quant_ctx['universe_n']} "
                  f"(as of {quant_ctx.get('as_of','—')})  "
                  f"[value {st.get('value',0):+.2f}  quality {st.get('quality',0):+.2f}  momentum {st.get('momentum',0):+.2f}]")
        if quant_ctx.get("eps_fwd") is not None:
            gc = GREEN if quant_ctx["eps_growth"] > 0 else RED
            print(f"    12M earnings model: ${quant_ctx['eps_ttm']:.2f} → ${quant_ctx['eps_fwd']:.2f} "
                  f"({gc}{quant_ctx['eps_growth']*100:+.0f}%{RESET})")
        _d = quant_ctx.get("dcf_llm")
        if _d:
            dc = GREEN if _d["upside"] > 0 else RED
            print(f"    Lifecycle DCF ({_d['lifecycle']}): intrinsic ${_d['intrinsic']:.0f} → "
                  f"{dc}{_d['upside']*100:+.0f}%{RESET} (ke {_d['ke']*100:.1f}%)")
        if quant_ctx.get("traps"):
            print(f"    {YELLOW}⚠ trap guards: {', '.join(quant_ctx['traps'])}{RESET}")
    else:
        print(f"    {DIM}unavailable (outside Sharadar panel/SF1){RESET}")

    # Scanner lifecycle DCF joins the BLEND as a first-class valuation source (validated stack:
    # EPS-ML forward EPS + lifecycle assumptions). Traps reduce its confidence.
    scanner_src = None
    if quant_ctx:
        _sd = quant_ctx.get("dcf_llm") or quant_ctx.get("dcf_fixed")
        if _sd and _sd.get("upside") is not None:
            scanner_src = {"expected_return": float(_sd["upside"]),
                           "confidence": 0.35 if quant_ctx.get("dcf_llm") else 0.25}
            if quant_ctx.get("traps"):
                scanner_src["confidence"] *= 0.8


    # ── Valuation engine (selected by policy) + quant reliability, then metrics + Stage A ──
    try:
        import valuation_engines as _ve, policy as _polm
        _polE = all_data.get("policy") or _polm.ApplicabilityPolicy.for_issuer(all_data.get("issuer"))
        all_data["policy"] = _polE
        _eng_name = _polE.valuation_engine
        if (_eng_name == "MixedEquityValuation" and (all_data.get("segment_facts") or {}).get("business_line_profit")
                and (all_data.get("segments_canonical") or {}).get("status") == "RECONCILED"):
            _eng_name = "SumOfPartsValuation"
        _eng = _ve.select_engine(_eng_name)
        _ovE = all_data.get("overview") or {}
        _bqE = (((all_data.get("balance") or {}).get("quarterlyReports") or [{}])[0]) or {}
        _incE = ((all_data.get("income") or {}).get("quarterlyReports") or [])[:4]
        _taxE = None
        try:
            _pt = sum((safe_float(q.get("incomeBeforeTax")) or 0) for q in _incE); _tx = sum((safe_float(q.get("incomeTaxExpense")) or 0) for q in _incE)
            if _pt > 0 and 0 <= _tx / _pt < 0.5: _taxE = _tx / _pt
        except Exception:  # noqa: BLE001
            pass
        _ndE = (safe_float(_bqE.get("shortLongTermDebtTotal")) or safe_float(_bqE.get("longTermDebt")) or 0.0) - (safe_float(_bqE.get("cashAndCashEquivalentsAtCarryingValue")) or 0.0)
        all_data["valuation_engine"] = _eng.run(fair_val_result, {"segment_brief": all_data.get("segment_brief"), "segment_facts": all_data.get("segment_facts"),
                                                                  "segments_canonical": all_data.get("segments_canonical"),
                                                                 "shares": safe_float(_ovE.get("SharesOutstanding")), "tax_rate": _taxE, "net_debt": _ndE,
                                                                 "rf": canonical_risk_free(all_data), "erp": (all_data.get("damodaran_inputs") or {}).get("erp") or DAMODARAN_ERP})
        _ver = all_data["valuation_engine"]
        print(f"  [engine] {_ver['name']} → {('$%.2f' % _ver['value']) if _ver.get('value') else _ver['status']}" + (" · " + "; ".join(_ver["notes"]) if _ver.get("notes") else ""))
        all_data["quant_reliability"] = _polm.quant_reliability({"dir_acc": (ml_result or {}).get("backtest_dir_accuracy"), "rank_ic": (ml_result or {}).get("backtest_rank_ic"),
                                                                 "n_oos_eff": (ml_result or {}).get("n_oos_effective") or (ml_result or {}).get("n_oos_eff")})
        if ml_result is not None:
            ml_result["_reliability"] = all_data["quant_reliability"]
        print(f"  [quant] reliability {all_data['quant_reliability']['state']} — {all_data['quant_reliability']['basis']}; weight cap ×{all_data['quant_reliability']['weight_cap']:.2f}")
    except Exception as _ee:  # noqa: BLE001
        print(f"  {YELLOW}[engine] selection failed: {str(_ee)[:100]}{RESET}")
    try:
        import canonical as _cn3
        all_data["corp_states"] = [_cn3.corp_action_state(a) for a in (all_data.get("corp_actions") or [])]
        all_data["metrics"] = _cn3.build_metrics(all_data, pred_fund_result, fair_val_result, current_price)
        _prec = _cn3.reconcile_peer_table(all_data, all_data["metrics"])
        if _prec.get("dropped"):
            print("  [peers] removed " + ", ".join(x["ticker"] for x in _prec["dropped"]))
    except Exception as _pce:  # noqa: BLE001
        print(f"  {YELLOW}[peers] reconciliation failed: {str(_pce)[:100]}{RESET}")
    _panel_rel, _panel_row = _panel_state(ticker)
    all_data["ownership_13f"] = _ownership_13f(ticker)
    all_data["panel_reliability"] = _panel_rel
    try:
        import valuation_engines as _veM
        _eng_meta = (_veM.select_engine(_eng_name).metadata() if "_eng_name" in locals() else {})
    except Exception:  # noqa: BLE001
        _eng_meta = {}
    _fv = _prun.forecast_stage(pred_fund=pred_fund_result, fair_val=fair_val_result, story=story_val_result,
                              engine_run=all_data.get("valuation_engine"), engine_meta=_eng_meta, tech=tech, fund=fund,
                              sent=sent, macro=macro_result, ml=ml_result, quant_rel=all_data.get("quant_reliability"),
                              panel_rel=_panel_rel, panel_row=_panel_row, all_data=all_data)
    print(f"  [forecast] {'PASS' if _fv.passed else RED + 'FAIL' + RESET} — " + ", ".join(f"{c.id} {c.status}" for c in _fv.checks)
          + f"; coherence {(_prun.state.forecast.get('coherence') or {}).get('status')}")
    if not _fv.passed:
        for c in _fv.blocking:
            print(f"      !! {c.id} {c.name}: {c.detail[:160]}")
        _prun.write(_final_path)
        print(f"  [final] WITHHELD at forecast validation — the analyst panel was not run. Saved {_final_path}")
        return

    # ── ANALYST PANEL: structured judgement over the validated state; every figure is a token ──
    llm_result = None
    if args.llm:
        print(f"  Analyst Panel Debate (Neutral · Bear · Bull · PM)...")
        try:
            llm_result = analyze_llm_debate_yearly(
                ticker, current_price, tech, fund, sent, macro_result, ml_result,
                pred_fund=pred_fund_result, fair_val=fair_val_result, insider=all_data.get("insider"), data=all_data,
                verbose=args.verbose, state_mode=_prun.panel_inputs())
        except Exception as _dbe:  # noqa: BLE001 — an incomplete panel is withheld with a QA record, never a lost run
            import traceback as _tb
            _prun.notes.append(f"debate failed: {type(_dbe).__name__}: {str(_dbe)[:200]} | "
                               + _tb.format_exc()[-1200:].replace(chr(10), " / "))
            print(f"    {RED}debate failed: {type(_dbe).__name__}: {str(_dbe)[:120]}{RESET}")
            llm_result = None
        if not llm_result:
            print(f"    {DIM}unavailable{RESET}")
    _seat_prose = _prun.scenarios_from_panel(llm_result)

    # ── BLEND → ANALYSIS_FINALIZED (the state freezes; its hash is the analysis version) ──
    print(f"\n{BOLD}[4/5] Blending Predictions{RESET}")
    blend = blend_predictions_yearly(fair_val_result, macro_result, ml_result, _prun.llm_shaped(), scanner=scanner_src)
    _xv = _prun.finalize(blend)
    print(f"  [analysis] {'PASS' if _xv.passed else RED + 'FAIL' + RESET} — " + ", ".join(f"{c.id} {c.status}" for c in _xv.checks))
    if not _xv.passed:
        for c in _xv.blocking:
            print(f"      !! {c.id} {c.name}: {c.detail[:160]}")
        _prun.write(_final_path)
        print(f"  [final] WITHHELD at analysis validation. Saved {_final_path}")
        return
    print(f"  [analysis] frozen · version {_prun.state.version} · {_prun.state.value('recommendation.rating')} "
          f"{_prun.state.value('blend.headline_return'):+.3f}")

    # ── REPORT: summary + one-pass prose ingestion → UserReport → render → REPORT_VALIDATED ──
    print(f"\n{BOLD}[5/5] Composing the report{RESET}")
    _pub = _prun.compose_safe(_seat_prose)
    _ra = _prun.report_audit
    if _ra is not None:
        print(f"  [report] {'PASS' if _ra.passed else RED + 'FAIL' + RESET} — " + ", ".join(f"{c.id} {c.status}" for c in _ra.checks))
        for c in _ra.blocking:
            print(f"      !! {c.id} {c.name}: {c.detail[:200]}")
    print(f"  [prose] {len(_prun.rejections)} sentence(s) removed at ingestion")
    _prun.write(_final_path, _pub)
    print(f"  [final] {'PUBLISHED' if _pub else 'WITHHELD'} — saved {_final_path}")


if __name__ == "__main__":
    main()