Delete train.py

train.py

@@ -1,345 +0,0 @@
-"""
-train.py — Full training pipeline. Run this script to train the model.
-
-Usage:
-    python train.py --symbols BTC-USDT ETH-USDT SOL-USDT ... --bars 500
-    python train.py --use-defaults --bars 300
-    python train.py --data-dir ./historical_csv   # load pre-saved CSVs
-
-Pipeline:
-    1. Fetch OHLCV for all symbols
-    2. Run rule engine to extract features (no lookahead)
-    3. Label each signal bar with forward-looking outcome
-    4. Concatenate all symbols (adds cross-asset diversity)
-    5. Walk-forward validation → choose threshold
-    6. Final model fit on full dataset
-    7. Save model + threshold + feature importances
-"""
-
-import argparse
-import json
-import logging
-import sys
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-
-sys.path.insert(0, str(Path(__file__).parent))
-
-from config import DEFAULT_SYMBOLS, TIMEFRAME, CANDLE_LIMIT
-from data_fetcher import fetch_multiple
-from regime import detect_regime
-from volume_analysis import analyze_volume
-from scorer import compute_structure_score, score_token
-from veto import apply_veto
-from feature_builder import build_feature_dict, validate_features
-from labeler import label_dataframe, compute_label_stats
-from walk_forward import run_walk_forward, summarize_walk_forward
-from model_backend import ModelBackend
-from ml_config import (
-    ML_DIR,
-    MODEL_PATH,
-    THRESHOLD_PATH,
-    FEATURE_IMP_PATH,
-    LABEL_PATH,
-    LGBM_PARAMS,
-    FEATURE_COLUMNS,
-    LABEL_FORWARD_BARS,
-    THRESHOLD_MIN,
-    THRESHOLD_MAX,
-    THRESHOLD_STEPS,
-    THRESHOLD_OBJECTIVE,
-    STOP_MULT,
-)
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
-    stream=sys.stdout,
-)
-logger = logging.getLogger("train")
-
-
-def infer_direction(trend: str, breakout: int) -> int:
-    if trend == "bullish" or breakout == 1:
-        return 1
-    if trend == "bearish" or breakout == -1:
-        return -1
-    return 0
-
-
-def extract_features_and_labels(
-    symbol: str,
-    df: pd.DataFrame,
-) -> pd.DataFrame:
-    """
-    Run the full rule engine over a DataFrame, bar by bar (forward-only).
-    Returns a DataFrame with feature columns + 'label' + 'direction' + 'timestamp'.
-
-    Implementation note: we compute regime/volume/scores using the full
-    historical series up to each bar — no information from future bars
-    is ever used. The label is computed separately using FORWARD bars only.
-    """
-    if len(df) < 60:
-        logger.warning(f"{symbol}: too short ({len(df)} bars), skipping")
-        return pd.DataFrame()
-
-    # Compute full-series regime and volume (these use only past data internally)
-    try:
-        regime_data  = detect_regime(df)
-        volume_data  = analyze_volume(df, atr_series=regime_data["atr_series"])
-    except Exception as e:
-        logger.error(f"{symbol}: rule engine error: {e}")
-        return pd.DataFrame()
-
-    atr_series = regime_data["atr_series"]
-
-    # Build per-bar feature rows for all bars with valid ATR (skip first ATR_PERIOD)
-    rows = []
-    n = len(df)
-
-    for i in range(30, n):
-        # Slice up to bar i (inclusive) — simulate running bar by bar
-        df_i = df.iloc[: i + 1]
-
-        try:
-            r_i = detect_regime(df_i)
-            v_i = analyze_volume(df_i, atr_series=r_i["atr_series"])
-        except Exception:
-            continue
-
-        sc_i = compute_structure_score(r_i)
-        direction = infer_direction(r_i["trend"], v_i["breakout"])
-        vetoed, _ = apply_veto(r_i, v_i, sc_i, direction=direction)
-
-        # Only label bars that the rule engine would have flagged as signals
-        is_signal = not vetoed and r_i["regime_confidence"] > 0.3
-
-        scores = score_token(r_i, v_i, vetoed=False)  # compute scores even if vetoed
-
-        try:
-            feat = build_feature_dict(r_i, v_i, scores)
-        except (KeyError, ValueError):
-            continue
-
-        if not validate_features(feat):
-            continue
-
-        feat["_symbol"]    = symbol
-        feat["_bar_idx"]   = i
-        feat["_timestamp"] = df.index[i]
-        feat["_is_signal"] = int(is_signal)
-        feat["_direction"] = direction
-        feat["_atr"]       = float(r_i["atr"])
-        rows.append(feat)
-
-    if not rows:
-        return pd.DataFrame()
-
-    result = pd.DataFrame(rows)
-
-    # Label: compute forward outcomes for signal bars
-    signal_mask_full = pd.Series(False, index=df.index)
-    direction_full   = pd.Series(0, index=df.index)
-    atr_full         = atr_series
-
-    for row in rows:
-        if row["_is_signal"]:
-            idx = df.index[row["_bar_idx"]]
-            signal_mask_full[idx] = True
-            direction_full[idx]   = row["_direction"]
-
-    labels = label_dataframe(
-        df=df,
-        signal_mask=signal_mask_full,
-        atr_series=atr_full,
-        direction_series=direction_full,
-        forward_bars=LABEL_FORWARD_BARS,
-    )
-
-    # Merge labels back into result
-    result = result.set_index("_timestamp")
-    result["label"] = labels.reindex(result.index)
-    result = result.reset_index().rename(columns={"index": "_timestamp"})
-
-    # Keep only signal bars with valid labels
-    result = result[result["_is_signal"] == 1].copy()
-    result = result.dropna(subset=["label"])
-    result["label"] = result["label"].astype(int)
-
-    logger.info(
-        f"{symbol}: {len(result)} labeled signals — "
-        f"wr={result['label'].mean():.3f}"
-    )
-    return result
-
-
-def build_dataset(
-    symbols: list,
-    bars: int = CANDLE_LIMIT,
-    data_dir: Path = None,
-) -> pd.DataFrame:
-    """Fetch data and build full labeled feature dataset."""
-    all_frames = []
-
-    if data_dir and data_dir.exists():
-        logger.info(f"Loading CSVs from {data_dir}")
-        for csv_path in sorted(data_dir.glob("*.csv")):
-            sym = csv_path.stem
-            df = pd.read_csv(csv_path, index_col=0, parse_dates=True)
-            df.index = pd.to_datetime(df.index, utc=True)
-            df.sort_index(inplace=True)
-            frame = extract_features_and_labels(sym, df)
-            if not frame.empty:
-                all_frames.append(frame)
-    else:
-        logger.info(f"Fetching OHLCV for {len(symbols)} symbols ({bars} bars each)")
-        ohlcv_map = fetch_multiple(symbols, limit=bars, min_bars=60)
-        for sym, df in ohlcv_map.items():
-            frame = extract_features_and_labels(sym, df)
-            if not frame.empty:
-                all_frames.append(frame)
-
-    if not all_frames:
-        raise ValueError("No labeled data produced. Check symbols and API connectivity.")
-
-    combined = pd.concat(all_frames, ignore_index=True)
-    combined.sort_values("_timestamp", inplace=True)
-    combined.reset_index(drop=True, inplace=True)
-    logger.info(
-        f"Dataset: {len(combined)} samples across {len(all_frames)} symbols | "
-        f"overall wr={combined['label'].mean():.3f}"
-    )
-    return combined
-
-
-def fit_final_model(
-    X: np.ndarray,
-    y: np.ndarray,
-    params: dict,
-    val_frac: float = 0.15,
-) -> ModelBackend:
-    """Fit final model on full dataset with internal validation split."""
-    split = int(len(X) * (1 - val_frac))
-    X_tr, y_tr = X[:split], y[:split]
-    X_va, y_va = X[split:], y[split:]
-
-    pos_frac = y_tr.mean()
-    sample_weight = None
-    if 0.05 < pos_frac < 0.95:
-        sample_weight = np.where(y_tr == 1, 1.0 / pos_frac, 1.0 / (1 - pos_frac))
-
-    backend = ModelBackend(params=params, calibrate=True)
-    backend.fit(X_tr, y_tr, X_va, y_va, sample_weight=sample_weight)
-    logger.info(f"Final model: {backend.n_iter_} boosting rounds, backend={backend.backend_name}")
-    return backend
-
-
-def save_artifacts(
-    backend: ModelBackend,
-    threshold: float,
-    summary: dict,
-    dataset: pd.DataFrame,
-):
-    import joblib
-
-    ML_DIR.mkdir(parents=True, exist_ok=True)
-
-    # Save model
-    joblib.dump(backend, MODEL_PATH)
-    logger.info(f"Model saved → {MODEL_PATH}")
-
-    # Save threshold
-    thresh_data = {
-        "threshold": threshold,
-        "objective": THRESHOLD_OBJECTIVE,
-        "n_folds_used": summary.get("n_folds", 0),
-        "mean_test_expectancy": summary.get("mean_expectancy"),
-        "mean_test_sharpe": summary.get("mean_sharpe"),
-        "mean_test_precision": summary.get("mean_precision"),
-    }
-    with open(THRESHOLD_PATH, "w") as f:
-        json.dump(thresh_data, f, indent=2)
-    logger.info(f"Threshold saved → {THRESHOLD_PATH} (value={threshold:.4f})")
-
-    # Save feature importances
-    imp_df = pd.DataFrame({
-        "feature": FEATURE_COLUMNS,
-        "importance": backend.feature_importances_,
-    }).sort_values("importance", ascending=False)
-    imp_df.to_csv(FEATURE_IMP_PATH, index=False)
-    logger.info(f"Feature importances saved → {FEATURE_IMP_PATH}")
-
-    # Save label stats
-    label_stats = compute_label_stats(pd.Series(dataset["label"].values))
-    with open(LABEL_PATH, "w") as f:
-        json.dump(label_stats, f, indent=2)
-    logger.info(f"Label stats: {label_stats}")
-
-
-def main(args):
-    logger.info("=" * 60)
-    logger.info("OKX TRADE FILTER — TRAINING PIPELINE")
-    logger.info("=" * 60)
-
-    if args.use_defaults:
-        symbols = DEFAULT_SYMBOLS
-    elif args.symbols:
-        symbols = args.symbols
-    else:
-        symbols = DEFAULT_SYMBOLS[:20]  # safe default for quick runs
-
-    data_dir = Path(args.data_dir) if args.data_dir else None
-    dataset  = build_dataset(symbols, bars=args.bars, data_dir=data_dir)
-
-    X = dataset[FEATURE_COLUMNS].values.astype(np.float64)
-    y = dataset["label"].values.astype(np.int32)
-    timestamps = dataset["_timestamp"].values
-
-    logger.info(f"Feature matrix: {X.shape} | Positive rate: {y.mean():.4f}")
-
-    # Walk-forward validation
-    logger.info("Running walk-forward validation...")
-    wf_results = run_walk_forward(X, y, timestamps=timestamps, params=LGBM_PARAMS)
-    summary    = summarize_walk_forward(wf_results)
-
-    logger.info("\n=== WALK-FORWARD SUMMARY ===")
-    logger.info(f"  Folds:            {summary['n_folds']}")
-    logger.info(f"  Mean threshold:   {summary['mean_threshold']:.4f} ± {summary['std_threshold']:.4f}")
-    logger.info(f"  Mean expectancy:  {summary['mean_expectancy']}")
-    logger.info(f"  Mean sharpe:      {summary['mean_sharpe']}")
-    logger.info(f"  Mean precision:   {summary['mean_precision']}")
-
-    if summary.get("mean_expectancy") is not None and summary["mean_expectancy"] < 0:
-        logger.warning("Negative mean expectancy! Model may not generalize. Check data quality.")
-
-    # Choose final threshold: mean of walk-forward optimal thresholds
-    final_threshold = summary["mean_threshold"]
-    logger.info(f"\nFinal threshold: {final_threshold:.4f}")
-
-    # Feature importance report
-    imp_arr = np.array(summary["avg_feature_importance"])
-    imp_pairs = sorted(zip(FEATURE_COLUMNS, imp_arr), key=lambda x: x[1], reverse=True)
-    logger.info("\n=== TOP 15 FEATURES BY IMPORTANCE ===")
-    for feat, imp in imp_pairs[:15]:
-        bar = "█" * int(imp / imp_arr.max() * 30) if imp_arr.max() > 0 else ""
-        logger.info(f"  {feat:<28} {imp:>8.2f}  {bar}")
-
-    # Fit final model on all data
-    logger.info("\nFitting final model on full dataset...")
-    final_backend = fit_final_model(X, y, LGBM_PARAMS, val_frac=0.15)
-
-    # Save everything
-    save_artifacts(final_backend, final_threshold, summary, dataset)
-    logger.info("\n✓ Training complete.")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Train OKX trade probability filter")
-    parser.add_argument("--symbols", nargs="+", default=None, help="Symbol list, e.g. BTC-USDT ETH-USDT")
-    parser.add_argument("--use-defaults", action="store_true", help="Use all DEFAULT_SYMBOLS from config")
-    parser.add_argument("--bars", type=int, default=300, help="OHLCV bars to fetch per symbol")
-    parser.add_argument("--data-dir", type=str, default=None, help="Directory of pre-saved CSV files")
-    args = parser.parse_args()
-    main(args)