Spaces:
Running
Running
File size: 3,320 Bytes
88d2f2a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | """CLI for the PolyglotAlpha v2 backtest framework.
Usage:
python scripts/run_backtest.py --n 100 --random-seed 42 \\
--out outputs/backtest/ --mock-llm
python scripts/run_backtest.py --n 20 --real-llm
Set ``--real-llm`` only if ``GEMINI_API_KEY`` or ``OPENROUTER_API_KEY``
is in the environment (the LLM layer falls back to the deterministic
``MockLLM`` if no key is set).
"""
from __future__ import annotations
import argparse
import logging
import sys
import time
from pathlib import Path
# Allow running this file directly without an editable install.
_REPO_ROOT = Path(__file__).resolve().parents[1]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
from polyglot_alpha.backtest.runner import ( # noqa: E402
DEFAULT_OUTPUT_DIR,
DEFAULT_RESOLVED_PARQUET,
run_backtest,
)
LOGGER = logging.getLogger("polyglot_alpha.backtest.cli")
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--n", type=int, default=100, help="Number of markets to backtest.")
parser.add_argument(
"--random-seed", type=int, default=42, help="Seed for reproducibility."
)
parser.add_argument(
"--out",
type=Path,
default=DEFAULT_OUTPUT_DIR,
help="Output directory for results (default: outputs/backtest/).",
)
parser.add_argument(
"--resolved-parquet",
type=Path,
default=DEFAULT_RESOLVED_PARQUET,
help=f"Path to resolved markets parquet (default: {DEFAULT_RESOLVED_PARQUET}).",
)
mode = parser.add_mutually_exclusive_group()
mode.add_argument(
"--mock-llm",
action="store_true",
default=True,
help="Use deterministic mock LLM (default; fast).",
)
mode.add_argument(
"--real-llm",
action="store_false",
dest="mock_llm",
help="Use real LLM via existing make_llm() (slower; needs API key).",
)
parser.add_argument(
"--no-embeddings",
action="store_true",
help="Force the deterministic Jaccard similarity fallback.",
)
parser.add_argument("--verbose", action="store_true", help="Enable INFO logging.")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = _parse_args(argv)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
started = time.time()
summary = run_backtest(
n=args.n,
seed=args.random_seed,
output_dir=args.out,
mock_llm=args.mock_llm,
use_embeddings=not args.no_embeddings,
parquet_path=args.resolved_parquet,
)
elapsed = time.time() - started
print(
f"Backtest done in {elapsed:.1f}s | n={summary.get('n_markets', 0)} | "
f"accuracy={summary.get('outcome_accuracy', 0) * 100:.1f}% | "
f"ROI=${summary.get('estimated_total_roi_usdc', 0):,.2f} | "
f"PASS={summary.get('n_PASS', 0)} FAIL={summary.get('n_FAIL', 0)} "
f"BORDERLINE={summary.get('n_BORDERLINE', 0)}"
)
print(f"Artifacts: {args.out}")
return 0
if __name__ == "__main__": # pragma: no cover
sys.exit(main())
|