"""Turn a pile of statistics into one number and one sentence. The Reality Score is deliberately harsh. Most published backtests would score below 40, and that is the point: the score exists to be screenshotted. """ from __future__ import annotations from typing import Dict, List, Optional import numpy as np __all__ = ["reality_score", "GRADES"] GRADES = [ (85, "A", "Survives everything we threw at it"), (70, "B", "Probably a real edge, with caveats"), (55, "C", "Ambiguous — could go either way"), (40, "D", "Mostly luck"), (0, "F", "Indistinguishable from randomness"), ] WEIGHTS = { "significance": 0.30, "selection": 0.25, "walk_forward": 0.20, "overfitting": 0.15, "robustness": 0.10, } def _ramp(value: float, good: float, bad: float) -> float: """Linear 0-100 score where ``good`` maps to 100 and ``bad`` maps to 0.""" if not np.isfinite(value): return 50.0 if good == bad: return 50.0 scaled = (value - bad) / (good - bad) return float(np.clip(scaled, 0.0, 1.0) * 100.0) def reality_score( metrics: Dict[str, float], benchmark_metrics: Dict[str, float], p_value: Optional[float] = None, dsr: Optional[float] = None, pbo: Optional[float] = None, wf_efficiency: Optional[float] = None, wf_win_rate: Optional[float] = None, cost_stress_ratio: Optional[float] = None, benchmark_correlation: Optional[float] = None, attribution: Optional[Dict[str, object]] = None, survivorship: Optional[object] = None, benchmark_name: str = "Buy & hold", permutation_label: str = "shuffled, structure-free markets", ) -> Dict[str, object]: """Combine the validation panel into a 0-100 score, a grade and warnings. ``attribution`` and ``survivorship`` are the portfolio-only inputs; both are optional so the single-asset path is unaffected. """ components: Dict[str, float] = {} components["significance"] = _ramp(p_value, good=0.01, bad=0.50) if p_value is not None else 50.0 components["selection"] = float(np.clip(dsr, 0.0, 1.0) * 100.0) if dsr is not None else 50.0 if wf_efficiency is not None: wf = _ramp(wf_efficiency, good=0.8, bad=-0.2) if wf_win_rate is not None: wf = 0.7 * wf + 0.3 * float(np.clip(wf_win_rate, 0.0, 1.0) * 100.0) components["walk_forward"] = wf else: components["walk_forward"] = 50.0 components["overfitting"] = _ramp(pbo, good=0.05, bad=0.50) if pbo is not None and np.isfinite(pbo) else 50.0 components["robustness"] = ( _ramp(cost_stress_ratio, good=0.8, bad=0.0) if cost_stress_ratio is not None else 50.0 ) score = float(sum(components[k] * w for k, w in WEIGHTS.items())) flags: List[str] = [] n_trades = int(metrics.get("n_trades", 0)) sharpe = float(metrics.get("sharpe", 0.0)) total_return = float(metrics.get("total_return", 0.0)) max_dd = float(metrics.get("max_drawdown", 0.0)) bench_sharpe = float(benchmark_metrics.get("sharpe", 0.0)) # The score asks "is the measured edge real?" — if the strategy lost money, # there is no edge to validate, whatever the statistics say about it. if total_return <= 0: flags.append( f"The strategy lost money over the test period ({total_return:.1%}). " "There is no edge here to validate." ) score = min(score, 50.0) if total_return <= 0 < sharpe: flags.append( "Positive Sharpe with a negative total return: the average bar was profitable but " "the compounding was not. Volatility drag ate the arithmetic edge." ) if max_dd < -0.5: flags.append( f"Peak-to-trough drawdown of {max_dd:.0%} — an account running this would have been " "closed long before the recovery arrived." ) score = min(score, 60.0) if n_trades < 20: flags.append( f"Only {n_trades} position changes — with this few decisions, " "the result is a handful of coin flips, not a track record." ) score = min(score, 55.0) if p_value is not None and p_value > 0.10: flags.append( f"Permutation p-value is {p_value:.2f}: roughly {p_value * 100:.0f}% of " f"{permutation_label} did this well or better." ) if dsr is not None and dsr < 0.5: flags.append( f"Deflated Sharpe is {dsr:.2f} — once you account for how many variants were tried, " "the edge does not clear the selection-bias bar." ) if pbo is not None and np.isfinite(pbo) and pbo > 0.3: flags.append( f"Probability of backtest overfitting is {pbo:.0%}: the in-sample winner " "usually lands in the bottom half out of sample." ) if wf_efficiency is not None and wf_efficiency < 0.3: flags.append( f"Walk-forward efficiency is {wf_efficiency:.0%} — most of the in-sample Sharpe " "does not survive re-tuning and trading forward." ) if cost_stress_ratio is not None and cost_stress_ratio < 0.5: flags.append( "Tripling trading costs removes more than half the Sharpe. The edge is " "smaller than the friction it has to pay." ) if benchmark_correlation is not None and benchmark_correlation > 0.95: flags.append( f"Returns are {benchmark_correlation:.0%} correlated with {benchmark_name.lower()} — " "this is mostly a repackaged long position." ) if sharpe < bench_sharpe: flags.append( f"{benchmark_name} beat it on risk-adjusted return " f"({bench_sharpe:.2f} vs {sharpe:.2f} Sharpe)." ) if float(metrics.get("turnover_ann", 0.0)) > 100: flags.append( f"Annual turnover of {metrics.get('turnover_ann', 0):.0f}x is far beyond what " "retail execution can absorb without moving the modelled fills." ) # Portfolio-only checks. Style exposure you could buy in an ETF is not # alpha, and a universe with no failures in it is not a universe. if attribution and attribution.get("available"): if not attribution.get("alpha_significant"): flags.append( f"Style regression leaves no significant alpha (t = " f"{attribution.get('alpha_t_stat', 0):.1f}); the factors explain " f"{attribution.get('r_squared', 0):.0%} of returns" + ( f", mostly {attribution['dominant_factor']} exposure." if attribution.get("dominant_factor") else "." ) ) score = min(score, 65.0) if float(attribution.get("r_squared", 0.0)) > 0.9: flags.append( "Over 90% of the return variation is explained by simple style factors — " "this book is a repackaged index." ) if survivorship is not None and getattr(survivorship, "biased", False): flags.append(getattr(survivorship, "note", "Universe appears survivorship-biased.")) score = min(score, 60.0) score = float(np.clip(score, 0.0, 100.0)) grade, headline = next((g, h) for threshold, g, h in GRADES if score >= threshold) if score >= 70: verdict = ( f"Grade {grade}. {headline}. The edge is still there after the permutation test, " "after charging for every variant tried, and after walking it forward." ) elif score >= 55: verdict = ( f"Grade {grade}. {headline}. Parts of the panel hold up and parts do not — " "this is the zone where more data, not more tuning, is what settles it." ) elif score >= 40: verdict = ( f"Grade {grade}. {headline}. The backtest looks better than the evidence supports; " "the gap between the two is selection bias." ) else: verdict = ( f"Grade {grade}. {headline}. The null — {permutation_label} — produces results " "like this often enough that there is nothing here to trade." ) return { "score": round(score, 1), "grade": grade, "headline": headline, "verdict": verdict, "components": {k: round(v, 1) for k, v in components.items()}, "flags": flags, }