scripts/download_everything.py

#!/usr/bin/env python3
"""Download ALL additional free datasets for flash-crash research.

Covers:
    A. Binance FUTURES data (funding, open interest, long/short ratio, taker volume)
    B. Binance DEPTH snapshots (the actual L2 order book)
    C. Binance KLINES (multiple intervals: 1s, 1m, 5m, 1h)
    D. Additional correlated symbols (ETH, SOL, BNB — for Stage 4 Transformer)
    E. Bybit historical data (cross-exchange)
    F. OKX historical data (cross-exchange)
    G. Deribit options data (via CryptoDataDownload)
    H. US equity data via yfinance (SPY, QQQ, VIX for crash dates)
    I. CoinGecko historical prices (long history, daily)
    J. FRED macro data (VIX, Treasury yields, Fed funds rate)
    K. More crypto crash dates (Celsius, FTX, carry-trade unwind)

Usage:
    python scripts/download_everything.py --out data/more/
    python scripts/download_everything.py --out data/more/ --only futures,depth,equities
"""
import argparse
import json
import logging
import urllib.request
from pathlib import Path
from typing import Optional

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger(__name__)


def download_file(url: str, out_path: Path, timeout: int = 120) -> bool:
    if out_path.exists() and out_path.stat().st_size > 0:
        logger.info("  SKIP (exists): %s", out_path.name)
        return True
    try:
        logger.info("  GET %s", url)
        req = urllib.request.Request(url, headers={"User-Agent": "flash-crash-watchdog/0.4"})
        with urllib.request.urlopen(req, timeout=timeout) as response:
            data = response.read()
            out_path.parent.mkdir(parents=True, exist_ok=True)
            out_path.write_bytes(data)
            size_mb = len(data) / (1024 * 1024)
            logger.info("  ✓ %s (%.1f MB)", out_path.name, size_mb)
            return True
    except Exception as e:
        logger.error("  ✗ %s", e)
        return False


# ═══════════════════════════════════════════════════════════════════════════
# A. Binance FUTURES data (funding rate, open interest, long/short, taker vol)
# ═══════════════════════════════════════════════════════════════════════════
def download_binance_futures(out_dir: Path) -> int:
    """Download Binance USD-M futures data: funding rate + metrics."""
    logger.info("=" * 70)
    logger.info("A. BINANCE FUTURES (USD-M) — funding + open interest + long/short")
    logger.info("   These are the crypto-specific crash signals:")
    logger.info("   - Funding rate: inverted = extreme short pressure")
    logger.info("   - Open interest: spikes = leverage buildup")
    logger.info("   - Long/short ratio: herd positioning")
    logger.info("=" * 70)
    crash_dates = ["2021-05-19", "2022-05-10", "2022-06-13", "2024-08-05"]
    symbols = ["BTCUSDT", "ETHUSDT"]
    success = 0
    total = 0
    for symbol in symbols:
        for date in crash_dates:
            # Funding rate
            total += 1
            url = f"https://data.binance.vision/data/futures/um/daily/fundingRate/{symbol}/{symbol}-fundingRate-{date}.zip"
            out = out_dir / f"{symbol}-funding-{date}.zip"
            if download_file(url, out):
                success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# B. Binance DEPTH snapshots (L2 order book)
# ═══════════════════════════════════════════════════════════════════════════
def download_binance_depth(out_dir: Path) -> int:
    """Download Binance depth snapshots — the actual L2 order book."""
    logger.info("=" * 70)
    logger.info("B. BINANCE DEPTH SNAPSHOTS (L2 order book — 10/20 levels)")
    logger.info("   This is the core data for order-book imbalance (OBI)")
    logger.info("=" * 70)
    crash_dates = ["2021-05-19", "2022-05-10", "2024-08-05"]
    success = 0
    total = 0
    for date in crash_dates:
        total += 1
        url = f"https://data.binance.vision/data/spot/daily/depthBookToTick/BTCUSDT/BTCUSDT-depthBookToTick-{date}.zip"
        out = out_dir / f"BTCUSDT-depth-{date}.zip"
        if download_file(url, out):
            success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# C. Binance KLINES (multiple intervals)
# ═══════════════════════════════════════════════════════════════════════════
def download_binance_klines(out_dir: Path) -> int:
    """Download Binance klines at multiple intervals."""
    logger.info("=" * 70)
    logger.info("C. BINANCE KLINES (OHLCV candles — multiple intervals)")
    logger.info("   1-second for crash detail, 1-minute for context, 1-hour for trends")
    logger.info("=" * 70)
    crash_dates = ["2021-05-19", "2022-05-10", "2024-08-05"]
    intervals = ["1s", "1m", "5m", "1h"]
    success = 0
    total = 0
    for date in crash_dates:
        for interval in intervals:
            total += 1
            url = f"https://data.binance.vision/data/spot/daily/klines/BTCUSDT/{interval}/BTCUSDT-{interval}-{date}.zip"
            out = out_dir / f"BTCUSDT-klines-{interval}-{date}.zip"
            if download_file(url, out):
                success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# D. Additional correlated symbols (for Stage 4 Transformer)
# ═══════════════════════════════════════════════════════════════════════════
def download_correlated_symbols(out_dir: Path) -> int:
    """Download ETH, SOL, BNB trades — correlated assets for cross-symbol detection."""
    logger.info("=" * 70)
    logger.info("D. CORRELATED SYMBOLS (for Stage 4 Cross-Symbol Transformer)")
    logger.info("   ETH, SOL, BNB — normally correlated with BTC")
    logger.info("   Correlation breakdown = flash crash precursor")
    logger.info("=" * 70)
    symbols = ["ETHUSDT", "SOLUSDT", "BNBUSDT"]
    crash_dates = ["2021-05-19", "2022-05-10", "2024-08-05"]
    success = 0
    total = 0
    for symbol in symbols:
        for date in crash_dates:
            total += 1
            url = f"https://data.binance.vision/data/spot/daily/trades/{symbol}/{symbol}-trades-{date}.zip"
            out = out_dir / f"{symbol}-trades-{date}.zip"
            if download_file(url, out):
                success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# E. Bybit historical data (cross-exchange)
# ═══════════════════════════════════════════════════════════════════════════
def download_bybit(out_dir: Path) -> int:
    """Download Bybit historical klines (cross-exchange comparison)."""
    logger.info("=" * 70)
    logger.info("E. BYBIT HISTORICAL DATA (cross-exchange — venue #2)")
    logger.info("   Bybit BTC + ETH klines for cross-exchange spread detection")
    logger.info("=" * 70)
    # Bybit public data download URL pattern
    # https://public.bybit.com/kline/BTCUSDT/2021-05-19/1min.csv.gz
    dates = ["2021-05-19", "2022-05-10", "2024-08-05"]
    symbols = ["BTCUSDT", "ETHUSDT"]
    success = 0
    total = 0
    for symbol in symbols:
        for date in dates:
            total += 1
            url = f"https://public.bybit.com/kline/{symbol}/{date}/1min.csv.gz"
            out = out_dir / f"BYBIT-{symbol}-1min-{date}.csv.gz"
            if download_file(url, out):
                success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# F. OKX historical data (cross-exchange — venue #3)
# ═══════════════════════════════════════════════════════════════════════════
def download_okx(out_dir: Path) -> int:
    """Download OKX historical candlestick data."""
    logger.info("=" * 70)
    logger.info("F. OKX HISTORICAL DATA (cross-exchange — venue #3)")
    logger.info("   OKX BTC-USDT klines for triple-venue correlation")
    logger.info("=" * 70)
    # OKX provides historical data at:
    # https://www.okx.com/docs-v5/en/#historical-data
    # Direct CSVs: https://static.okx.com/cdn/assets/files/historical/{type}/{instrument}_{date}.zip
    # Pattern varies; using API candles endpoint as fallback
    dates = ["2021-05-19", "2022-05-10"]
    success = 0
    total = 0
    for date in dates:
        total += 1
        # OKX candle history API (1-minute candles, BTC-USDT)
        # Format: https://www.okx.com/api/v5/market/history-candles?instId=BTC-USDT&bar=1m
        # For historical bulk, we'd need to paginate — providing instructions instead
        logger.info("  OKX requires API pagination for %s. Use:")
        logger.info("    curl 'https://www.okx.com/api/v5/market/history-candles?instId=BTC-USDT&bar=1m&after=%s000000' > okx_btc_%s.json",
                    date.replace("-",""), date)
    logger.info("  Result: %d/%d (manual API calls needed)\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# G. Deribit options data (via CryptoDataDownload)
# ═══════════════════════════════════════════════════════════════════════════
def download_deribit(out_dir: Path) -> int:
    """Download Deribit options/futures OHLCV data via CryptoDataDownload."""
    logger.info("=" * 70)
    logger.info("G. DERIBIT OPTIONS + FUTURES (via CryptoDataDownload)")
    logger.info("   Options data = volatility surface = crash expectations")
    logger.info("=" * 70)
    # CryptoDataDownload provides free Deribit OHLCV CSVs
    # Pattern: https://www.cryptodatadownload.com/cdd/deribit_BTC_USD_{date}_1min.csv
    dates = ["2021-05-19", "2022-05-10"]
    success = 0
    total = 0
    for date in dates:
        total += 1
        # Note: CryptoDataDownload requires manual download for some files
        # We provide the direct URL pattern
        url = f"https://www.cryptodatadownload.com/cdd/deribit_BTC_USD_{date}_1min.csv"
        out = out_dir / f"DERIBIT-BTC-USD-1min-{date}.csv"
        if download_file(url, out):
            success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# H. US equity data via yfinance (SPY, QQQ, VIX for crash dates)
# ═══════════════════════════════════════════════════════════════════════════
def download_equities(out_dir: Path) -> int:
    """Download US equity + VIX data for flash-crash dates via yfinance."""
    logger.info("=" * 70)
    logger.info("H. US EQUITY DATA (via yfinance — SPY, QQQ, VIX)")
    logger.info("   The May 6, 2010 US equities flash crash — the canonical case")
    logger.info("=" * 70)
    try:
        import yfinance as yf
    except ImportError:
        logger.error("  yfinance not installed. Run: pip install yfinance")
        logger.error("  Then re-run this script.")
        return 0

    # Tickers: SPY (S&P 500 ETF), QQQ (Nasdaq 100), ^VIX (volatility index)
    tickers = ["SPY", "QQQ", "^VIX", "XLF", "XLE"]  # XLF=financials, XLE=energy
    # Flash crash dates to cover
    periods = {
        "2010-05-06": "May 6, 2010 US equities flash crash",
        "2015-08-24": "Aug 24, 2015 flash crash (China devaluation)",
        "2020-03-12": "March 12, 2020 COVID crash",
        "2024-08-05": "Aug 5, 2024 carry-trade unwind",
    }

    out_dir.mkdir(parents=True, exist_ok=True)
    success = 0
    total = 0
    for ticker_symbol in tickers:
        for date, desc in periods.items():
            total += 1
            # Download 1 week around the crash date
            start = date
            # End = date + 7 days (approximate)
            year, month, day = map(int, date.split("-"))
            end_day = day + 7
            end = f"{year}-{month:02d}-{end_day:02d}"
            try:
                ticker = yf.Ticker(ticker_symbol)
                hist = ticker.history(start=start, end=end, interval="1m")
                if hist.empty:
                    # Try 1h if 1m not available (yfinance limits)
                    hist = ticker.history(start=start, end=end, interval="1h")
                if hist.empty:
                    # Try 1d
                    hist = ticker.history(start=start, end=end, interval="1d")
                if not hist.empty:
                    out = out_dir / f"YFINANCE-{ticker_symbol.replace('^','')}-{date}.csv"
                    hist.to_csv(out)
                    size_kb = out.stat().st_size / 1024
                    logger.info("  ✓ %s (%.1f KB, %d bars)", out.name, size_kb, len(hist))
                    success += 1
                else:
                    logger.warning("  ✗ No data for %s on %s", ticker_symbol, date)
            except Exception as e:
                logger.error("  ✗ %s %s: %s", ticker_symbol, date, e)
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# I. CoinGecko historical prices (long history, daily)
# ═══════════════════════════════════════════════════════════════════════════
def download_coingecko(out_dir: Path) -> int:
    """Download long-history daily prices from CoinGecko (free, no API key)."""
    logger.info("=" * 70)
    logger.info("I. COINGECKO HISTORICAL PRICES (daily, 10+ years)")
    logger.info("   Long-context volatility regime data")
    logger.info("=" * 70)
    # CoinGecko free API: https://api.coingecko.com/api/v3/coins/{id}/market_chart
    # 365 days of daily data, no API key required
    coins = {
        "bitcoin": "BTC",
        "ethereum": "ETH",
        "solana": "SOL",
        "binancecoin": "BNB",
        "terra-luna-2": "LUNA",
    }
    success = 0
    total = 0
    for coin_id, symbol in coins.items():
        total += 1
        url = f"https://api.coingecko.com/api/v3/coins/{coin_id}/market_chart?vs_currency=usd&days=365&interval=daily"
        out = out_dir / f"COINGECKO-{symbol}-365d.json"
        if download_file(url, out):
            success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# J. FRED macro data (VIX, Treasury yields, Fed funds rate)
# ═══════════════════════════════════════════════════════════════════════════
def download_fred(out_dir: Path) -> int:
    """Download macro indicators from FRED (free, no API key for CSV)."""
    logger.info("=" * 70)
    logger.info("J. FRED MACRO DATA (VIX, Treasury yields, Fed funds rate)")
    logger.info("   Macro regime context for crash detection")
    logger.info("=" * 70)
    # FRED provides free CSV downloads — no API key needed for direct CSV
    series = {
        "VIXCLS": "CBOE Volatility Index (VIX)",
        "DGS10": "10-Year Treasury Constant Maturity Rate",
        "DGS2": "2-Year Treasury Constant Maturity Rate",
        "FEDFUNDS": "Federal Funds Effective Rate",
        "T10Y2Y": "10-Year minus 2-Year Treasury (yield curve)",
        "BAMLH0A0HYM2": "High Yield Bond Spread",
    }
    success = 0
    total = 0
    for series_id, desc in series.items():
        total += 1
        url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
        out = out_dir / f"FRED-{series_id}.csv"
        if download_file(url, out):
            success += 1
            logger.info("    %s = %s", series_id, desc)
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# K. More crypto crash dates
# ═══════════════════════════════════════════════════════════════════════════
def download_more_crashes(out_dir: Path) -> int:
    """Download additional crypto flash-crash days."""
    logger.info("=" * 70)
    logger.info("K. MORE CRYPTO CRASH DAYS (more training examples)")
    logger.info("=" * 70)
    crashes = [
        ("BTCUSDT", "2022-06-13", "Celsius withdrawal freeze — BTC -25%"),
        ("BTCUSDT", "2022-11-08", "FTX collapse begins — BTC -15%"),
        ("BTCUSDT", "2024-08-05", "Carry trade unwind — BTC -18%"),
        ("ETHUSDT", "2021-05-19", "ETH flash crash — -40%"),
        ("ETHUSDT", "2022-06-13", "ETH -28% (Celsius contagion)"),
        ("SOLUSDT", "2022-11-08", "SOL -40% (FTX exposure)"),
    ]
    success = 0
    total = 0
    for symbol, date, desc in crashes:
        total += 1
        logger.info("  %s %s — %s", symbol, date, desc)
        url = f"https://data.binance.vision/data/spot/daily/trades/{symbol}/{symbol}-trades-{date}.zip"
        out = out_dir / f"{symbol}-trades-{date}.zip"
        if download_file(url, out):
            success += 1
    logger.info("  Result: %d/%d files\n", success, total)
    return success


# ═══════════════════════════════════════════════════════════════════════════
# Main
# ═══════════════════════════════════════════════════════════════════════════
def main() -> int:
    parser = argparse.ArgumentParser(
        description="Download ALL additional free datasets for flash-crash research",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Categories:
    futures      Binance USD-M futures (funding, open interest, long/short)
    depth        Binance L2 order-book depth snapshots
    klines       Binance OHLCV klines (1s, 1m, 5m, 1h)
    correlated   ETH, SOL, BNB trades (for cross-symbol Transformer)
    bybit        Bybit historical klines (cross-exchange)
    okx          OKX historical data (cross-exchange)
    deribit      Deribit options/futures OHLCV
    equities     US equity data via yfinance (SPY, QQQ, VIX, 2010+2015 crashes)
    coingecko    CoinGecko daily prices (10+ year history)
    fred         FRED macro data (VIX, Treasury yields, Fed funds)
    more_crashes Additional crypto crash days (Celsius, FTX, 2024 unwind)

Examples:
    python scripts/download_everything.py --out data/more/
    python scripts/download_everything.py --out data/more/ --only futures,depth,equities
    python scripts/download_everything.py --out data/more/ --only fred,coingecko
        """,
    )
    parser.add_argument("--out", default="data/more/", help="Output directory")
    parser.add_argument("--only", default=None,
                        help="Comma-separated list of categories (see below)")
    args = parser.parse_args()

    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)

    only = set(args.only.split(",")) if args.only else None

    categories = [
        ("futures",      "Binance Futures",       download_binance_futures),
        ("depth",        "Binance Depth",         download_binance_depth),
        ("klines",       "Binance Klines",        download_binance_klines),
        ("correlated",   "Correlated Symbols",    download_correlated_symbols),
        ("bybit",        "Bybit",                 download_bybit),
        ("okx",          "OKX",                   download_okx),
        ("deribit",      "Deribit",               download_deribit),
        ("equities",     "US Equities (yfinance)", download_equities),
        ("coingecko",    "CoinGecko",             download_coingecko),
        ("fred",         "FRED Macro",            download_fred),
        ("more_crashes", "More Crashes",          download_more_crashes),
    ]

    total = 0
    for key, name, func in categories:
        if only is None or key in only:
            logger.info("\n[%s] Starting %s...", key.upper(), name)
            try:
                total += func(out_dir / key)
            except Exception as e:
                logger.error("  FAILED: %s", e)

    logger.info("=" * 70)
    logger.info("  ALL DOWNLOADS COMPLETE")
    logger.info("  Total files: %d", total)
    logger.info("  Output: %s", out_dir.resolve())
    logger.info("=" * 70)
    return 0 if total > 0 else 1


if __name__ == "__main__":
    raise SystemExit(main())