| """Run the institutional eval on a strategy. |
| |
| Usage: |
| python run_eval.py --data /path/to/curated --symbol BTCUSDT --freq 1h \ |
| --strategy funding_carry --cpcv-blocks 6 --cpcv-k 2 |
| |
| Pipeline: |
| 1. load curated data, assert schema and no-lookahead invariants |
| 2. CPCV (N=6, K=2 default) with purge=label horizon, embargo=8 bars |
| 3. for each combination: train tuning-free strategy, test OOS |
| 4. reconstruct paths, report PSR/DSR/PBO/MinBTL + raw risk metrics |
| 5. record the source-hash manifest with every run |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| from crypto_eval.data import load_local, load_manifest |
| from crypto_eval.engine import run_backtest |
| from crypto_eval.strategies import STRATEGIES |
| from crypto_eval.cv import make_cpcv, make_walkforward |
| from crypto_eval.stats import eval_backtest, eval_trial_set |
|
|
| PERIODS_PER_YEAR = { |
| "1h": 24 * 365, "4h": 6 * 365, "1d": 365, "1m": 60 * 24 * 365, |
| } |
|
|
|
|
| def main() -> int: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--data", required=True) |
| ap.add_argument("--symbol", default="BTCUSDT") |
| ap.add_argument("--freq", default="1h") |
| ap.add_argument("--strategy", default="funding_carry") |
| ap.add_argument("--horizon", type=int, default=8, |
| help="label horizon in bars (default 8 = one funding interval)") |
| ap.add_argument("--embargo", type=int, default=8) |
| ap.add_argument("--cpcv-blocks", type=int, default=6) |
| ap.add_argument("--cpcv-k", type=int, default=2) |
| ap.add_argument("--out", default="eval_report.json") |
| args = ap.parse_args() |
|
|
| data = load_local(args.data, args.symbol, args.freq) |
| manifest = load_manifest(args.data) |
| strategy_fn = STRATEGIES[args.strategy] |
|
|
| n = len(data.bars) |
| print(f"[eval] {args.symbol} {args.freq}: {n:,} bars, " |
| f"{data.check_funding_coverage(args.freq):,} funding ticks in range") |
|
|
| splitter, combos, audit = make_cpcv( |
| data.bars.index, horizon=args.horizon, embargo=args.embargo, |
| freq=args.freq, n_blocks=args.cpcv_blocks, k=args.cpcv_k, |
| ) |
| print(f"[eval] CPCV: {len(combos)} combinations, " |
| f"{len(combos) * args.cpcv_blocks // args.cpcv_k} paths, audit OK") |
|
|
| |
| positions = strategy_fn(data.bars, data.funding) |
| bt = run_backtest(data.bars, positions, data.funding) |
|
|
| oos_returns = [] |
| for j, (train_idx, test_idx) in enumerate(combos): |
| |
| |
| rets = bt["net_return"].to_numpy()[test_idx] |
| oos_returns.append(rets) |
|
|
| ppy = PERIODS_PER_YEAR[args.freq] |
| trial_stats = eval_trial_set(oos_returns) |
| per_combo = [] |
| for j, rets in enumerate(oos_returns): |
| s = eval_backtest( |
| rets, n_trials=1, trial_sr_variance=0.0, |
| periods_per_year=ppy, |
| ) |
| per_combo.append(s) |
|
|
| report = { |
| "symbol": args.symbol, |
| "freq": args.freq, |
| "strategy": args.strategy, |
| "n_bars": n, |
| "cpcv": {"n_blocks": args.cpcv_blocks, "k": args.cpcv_k, |
| "n_combinations": len(combos)}, |
| "purge_horizon": args.horizon, |
| "embargo": args.embargo, |
| "audit": {"ok": audit.ok, "violations": audit.violations}, |
| "data_manifest_sha256_inputs": sorted(set(manifest["outputs"].keys())), |
| "costs": { |
| "total_fees": bt["total_fees"], |
| "total_slippage": bt["total_slippage"], |
| "total_funding": bt["total_funding"], |
| "net_total_pnl": bt["net_total_pnl"], |
| }, |
| "trials": trial_stats, |
| "per_combination": per_combo, |
| "decision_rule": "accept only if deflated_sharpe > 0.95 and PBO <= 0.5", |
| } |
| Path(args.out).write_text(json.dumps(report, indent=2, default=str)) |
| print(f"[eval] wrote {args.out}") |
| print(f"[eval] PBO={trial_stats['probability_of_backtest_overfitting']:.3f} " |
| f"DSR(mean combo)=" |
| f"{np.mean([c['deflated_sharpe'] for c in per_combo if c['deflated_sharpe'] is not None]):.3f}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| sys.exit(main()) |
|
|
