tools/tool_ml_features.py

"""
Feature engineering and constants for the ML signal pipeline.

Exports path constants and feature-building functions used by both
the training pipeline (tool_ml_train.py) and prediction pipeline
(tool_ml_signal.py).
"""
import os
import warnings
import numpy as np
import pandas as pd

DATA_DIR    = os.path.join(os.path.dirname(__file__), "..", "data")

# Separate model files for ETF vs individual stock pipelines
ETF_MODEL_PATH    = os.path.join(DATA_DIR, "signal_model_etf.pkl")
ETF_SCALER_PATH   = os.path.join(DATA_DIR, "signal_scaler_etf.pkl")
ETF_SELECTOR_PATH = os.path.join(DATA_DIR, "signal_selector_etf.pkl")
ETF_META_PATH     = os.path.join(DATA_DIR, "signal_meta_etf.json")

STOCK_MODEL_PATH    = os.path.join(DATA_DIR, "signal_model_stock.pkl")
STOCK_SCALER_PATH   = os.path.join(DATA_DIR, "signal_scaler_stock.pkl")
STOCK_SELECTOR_PATH = os.path.join(DATA_DIR, "signal_selector_stock.pkl")
STOCK_META_PATH     = os.path.join(DATA_DIR, "signal_meta_stock.json")

# Legacy aliases — used by old code, now point to stock model
MODEL_PATH    = STOCK_MODEL_PATH
SCALER_PATH   = STOCK_SCALER_PATH
SELECTOR_PATH = STOCK_SELECTOR_PATH
META_PATH     = STOCK_META_PATH


# ── Feature engineering ───────────────────────────────────────────────────────

def _build_features(hist: pd.DataFrame, vix: pd.Series = None) -> pd.DataFrame:
    """
    Build feature matrix from OHLCV data.
    Every feature is normalised (ratios, percentages, z-scores) so the
    model generalises across different price levels and tickers.
    """
    close = hist["Close"]
    high  = hist["High"]
    low   = hist["Low"]
    vol   = hist["Volume"]

    df = pd.DataFrame(index=hist.index)

    # ── Momentum features ─────────────────────────────────────────────────────
    df["ret_1d"]  = close.pct_change(1)
    df["ret_5d"]  = close.pct_change(5)
    df["ret_10d"] = close.pct_change(10)
    df["ret_20d"] = close.pct_change(20)

    # RSI (14)
    delta = close.diff()
    gain  = delta.clip(lower=0).rolling(14).mean()
    loss  = (-delta.clip(upper=0)).rolling(14).mean()
    df["rsi"] = 100 - (100 / (1 + gain / loss.replace(0, np.nan)))

    # Rate of change
    df["roc_10"] = (close / close.shift(10)) - 1
    df["roc_20"] = (close / close.shift(20)) - 1

    # ── Trend features ────────────────────────────────────────────────────────
    df["dist_ma20"]  = (close / close.rolling(20).mean())  - 1
    df["dist_ma50"]  = (close / close.rolling(50).mean())  - 1
    df["dist_ma200"] = (close / close.rolling(200).mean()) - 1

    ema12 = close.ewm(span=12).mean()
    ema26 = close.ewm(span=26).mean()
    macd  = ema12 - ema26
    sig   = macd.ewm(span=9).mean()
    df["macd_hist_norm"] = (macd - sig) / close   # normalised by price

    # ── Volatility features ───────────────────────────────────────────────────
    ma20  = close.rolling(20).mean()
    std20 = close.rolling(20).std()
    df["bb_position"] = (close - (ma20 - 2*std20)) / (4 * std20.replace(0, np.nan))
    df["volatility"]  = close.pct_change().rolling(20).std()

    # ATR ratio
    tr    = pd.concat([high - low,
                       (high - close.shift()).abs(),
                       (low  - close.shift()).abs()], axis=1).max(axis=1)
    df["atr_ratio"] = tr.rolling(14).mean() / close

    # ── Volume features ───────────────────────────────────────────────────────
    vol_ma = vol.rolling(20).mean().replace(0, np.nan)
    df["vol_ratio"] = vol / vol_ma
    df["vol_trend"] = vol.rolling(5).mean() / vol_ma

    # ── Interaction features (new in v2) ──────────────────────────────────────
    # RSI × BB position: oversold AND near lower band = stronger signal
    df["rsi_x_bb"]    = df["rsi"] * (1 - df["bb_position"])
    # Volume × momentum: high volume + positive return = conviction
    df["vol_x_ret5"]  = df["vol_ratio"] * df["ret_5d"]

    # ── 52-week position features ─────────────────────────────────────────────
    df["dist_52w_high"] = (close / close.rolling(252).max())  - 1   # 0 = at high
    df["dist_52w_low"]  = (close / close.rolling(252).min())  - 1   # 0 = at low
    # Momentum vs market (using recent vs 6-month return)
    df["momentum_6m"] = (close / close.shift(126)) - 1

    # ── Market regime (VIX) ───────────────────────────────────────────────────
    if vix is not None:
        aligned = vix.reindex(df.index, method="ffill")
        df["vix_level"] = aligned / 20.0        # normalised: >1 = fear
        df["vix_trend"] = aligned.pct_change(5) # rising/falling fear

    return df.dropna()


def _build_labels(hist: pd.DataFrame,
                  forward_days: int = 10,
                  threshold: float  = 0.03) -> pd.Series:
    """
    Label each day based on what actually happened next.

    +1 = price rose > threshold in next forward_days  → BUY was right
    -1 = price fell > threshold                        → SELL was right
     0 = moved less than threshold                     → HOLD

    threshold=3% filters out noise. A signal that predicts 0.5% moves
    isn't tradeable after fees and spread.
    """
    close  = hist["Close"]
    fwd    = close.shift(-forward_days) / close - 1
    labels = pd.Series(0, index=fwd.index)
    labels[fwd >  threshold] =  1
    labels[fwd < -threshold] = -1
    return labels