Spaces:
Sleeping
Sleeping
File size: 6,108 Bytes
3be03dd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
Feature engineering and constants for the ML signal pipeline.
Exports path constants and feature-building functions used by both
the training pipeline (tool_ml_train.py) and prediction pipeline
(tool_ml_signal.py).
"""
import os
import warnings
import numpy as np
import pandas as pd
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
# Separate model files for ETF vs individual stock pipelines
ETF_MODEL_PATH = os.path.join(DATA_DIR, "signal_model_etf.pkl")
ETF_SCALER_PATH = os.path.join(DATA_DIR, "signal_scaler_etf.pkl")
ETF_SELECTOR_PATH = os.path.join(DATA_DIR, "signal_selector_etf.pkl")
ETF_META_PATH = os.path.join(DATA_DIR, "signal_meta_etf.json")
STOCK_MODEL_PATH = os.path.join(DATA_DIR, "signal_model_stock.pkl")
STOCK_SCALER_PATH = os.path.join(DATA_DIR, "signal_scaler_stock.pkl")
STOCK_SELECTOR_PATH = os.path.join(DATA_DIR, "signal_selector_stock.pkl")
STOCK_META_PATH = os.path.join(DATA_DIR, "signal_meta_stock.json")
# Legacy aliases — used by old code, now point to stock model
MODEL_PATH = STOCK_MODEL_PATH
SCALER_PATH = STOCK_SCALER_PATH
SELECTOR_PATH = STOCK_SELECTOR_PATH
META_PATH = STOCK_META_PATH
# ── Feature engineering ───────────────────────────────────────────────────────
def _build_features(hist: pd.DataFrame, vix: pd.Series = None) -> pd.DataFrame:
"""
Build feature matrix from OHLCV data.
Every feature is normalised (ratios, percentages, z-scores) so the
model generalises across different price levels and tickers.
"""
close = hist["Close"]
high = hist["High"]
low = hist["Low"]
vol = hist["Volume"]
df = pd.DataFrame(index=hist.index)
# ── Momentum features ─────────────────────────────────────────────────────
df["ret_1d"] = close.pct_change(1)
df["ret_5d"] = close.pct_change(5)
df["ret_10d"] = close.pct_change(10)
df["ret_20d"] = close.pct_change(20)
# RSI (14)
delta = close.diff()
gain = delta.clip(lower=0).rolling(14).mean()
loss = (-delta.clip(upper=0)).rolling(14).mean()
df["rsi"] = 100 - (100 / (1 + gain / loss.replace(0, np.nan)))
# Rate of change
df["roc_10"] = (close / close.shift(10)) - 1
df["roc_20"] = (close / close.shift(20)) - 1
# ── Trend features ────────────────────────────────────────────────────────
df["dist_ma20"] = (close / close.rolling(20).mean()) - 1
df["dist_ma50"] = (close / close.rolling(50).mean()) - 1
df["dist_ma200"] = (close / close.rolling(200).mean()) - 1
ema12 = close.ewm(span=12).mean()
ema26 = close.ewm(span=26).mean()
macd = ema12 - ema26
sig = macd.ewm(span=9).mean()
df["macd_hist_norm"] = (macd - sig) / close # normalised by price
# ── Volatility features ───────────────────────────────────────────────────
ma20 = close.rolling(20).mean()
std20 = close.rolling(20).std()
df["bb_position"] = (close - (ma20 - 2*std20)) / (4 * std20.replace(0, np.nan))
df["volatility"] = close.pct_change().rolling(20).std()
# ATR ratio
tr = pd.concat([high - low,
(high - close.shift()).abs(),
(low - close.shift()).abs()], axis=1).max(axis=1)
df["atr_ratio"] = tr.rolling(14).mean() / close
# ── Volume features ───────────────────────────────────────────────────────
vol_ma = vol.rolling(20).mean().replace(0, np.nan)
df["vol_ratio"] = vol / vol_ma
df["vol_trend"] = vol.rolling(5).mean() / vol_ma
# ── Interaction features (new in v2) ──────────────────────────────────────
# RSI × BB position: oversold AND near lower band = stronger signal
df["rsi_x_bb"] = df["rsi"] * (1 - df["bb_position"])
# Volume × momentum: high volume + positive return = conviction
df["vol_x_ret5"] = df["vol_ratio"] * df["ret_5d"]
# ── 52-week position features ─────────────────────────────────────────────
df["dist_52w_high"] = (close / close.rolling(252).max()) - 1 # 0 = at high
df["dist_52w_low"] = (close / close.rolling(252).min()) - 1 # 0 = at low
# Momentum vs market (using recent vs 6-month return)
df["momentum_6m"] = (close / close.shift(126)) - 1
# ── Market regime (VIX) ───────────────────────────────────────────────────
if vix is not None:
aligned = vix.reindex(df.index, method="ffill")
df["vix_level"] = aligned / 20.0 # normalised: >1 = fear
df["vix_trend"] = aligned.pct_change(5) # rising/falling fear
return df.dropna()
def _build_labels(hist: pd.DataFrame,
forward_days: int = 10,
threshold: float = 0.03) -> pd.Series:
"""
Label each day based on what actually happened next.
+1 = price rose > threshold in next forward_days → BUY was right
-1 = price fell > threshold → SELL was right
0 = moved less than threshold → HOLD
threshold=3% filters out noise. A signal that predicts 0.5% moves
isn't tradeable after fees and spread.
"""
close = hist["Close"]
fwd = close.shift(-forward_days) / close - 1
labels = pd.Series(0, index=fwd.index)
labels[fwd > threshold] = 1
labels[fwd < -threshold] = -1
return labels
|