#!/usr/bin/env python3 """C30: Sector peer return features. Hypothesis: Stocks within the same sector exhibit lead-lag relationships. TSMC often leads UMC by 1 trading day; Fubon leads Cathay and Mega. Adding yesterday's average peer return as a feature lets the model capture this intra-sector momentum signal. New features (non-leaking — all use shift(1)): peer_ret_1d — prior-day mean return of same-sector peers peer_ret_5d — 5-day return of peers ending yesterday All 12 stocks are loaded to compute peer returns even for the 5-stock eval set. Stocks with no peers (2412 telecom) get 0.0 fill. """ import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) import warnings; warnings.filterwarnings("ignore") import json import argparse import numpy as np import pandas as pd from scripts.improvement_harness import ( fetch_df, build_triple_barrier_labels, walk_forward, compute_metrics, CURRENT_FEATURES, DEFAULT_STOCKS, EXTENDED_STOCKS, PASS_DIR_ACC, PASS_UP_PREC, ) from models.predictor import _build_features SECTOR_MAP = { "semis": ["2330", "2454", "2303"], "electronics": ["2317", "2382", "2308"], "financials": ["2881", "2882", "2886"], "etfs": ["0050", "0056"], "telecom": ["2412"], } STOCK_SECTOR = {s: sec for sec, members in SECTOR_MAP.items() for s in members} C30_FEATURES = ["peer_ret_1d", "peer_ret_5d"] NEW_FEATURES = CURRENT_FEATURES + C30_FEATURES ALL_STOCKS = [s for members in SECTOR_MAP.values() for s in members] def _build_peer_close(stock_dfs: dict[str, pd.DataFrame]) -> dict[str, pd.Series]: """Return {stock_no: close Series indexed by YYYY-MM-DD string}.""" out = {} for s, df in stock_dfs.items(): if df is not None and not df.empty and "date" in df.columns: out[s] = df.set_index("date")["close"].astype(float) return out def _add_peer_features(feat: pd.DataFrame, df: pd.DataFrame, stock_no: str, peer_close: dict[str, pd.Series]) -> pd.DataFrame: sector = STOCK_SECTOR.get(stock_no, "") peers = [p for p in SECTOR_MAP.get(sector, []) if p != stock_no and p in peer_close] if not peers or "date" not in df.columns: feat["peer_ret_1d"] = 0.0 feat["peer_ret_5d"] = 0.0 return feat date_col = df["date"].astype(str).reset_index(drop=True) r1_cols, r5_cols = [], [] for p in peers: s = peer_close[p] r1 = s.pct_change(1).shift(1) # prior-day 1d return — non-leaking r5 = s.pct_change(5).shift(1) # 5d return ending yesterday — non-leaking r1_cols.append(date_col.map(r1.to_dict()).astype(float)) r5_cols.append(date_col.map(r5.to_dict()).astype(float)) feat = feat.reset_index(drop=True) feat["peer_ret_1d"] = pd.concat(r1_cols, axis=1).mean(axis=1).ffill().bfill().fillna(0.0).values feat["peer_ret_5d"] = pd.concat(r5_cols, axis=1).mean(axis=1).ffill().bfill().fillna(0.0).values return feat def main(): parser = argparse.ArgumentParser() parser.add_argument("--extended", action="store_true") args = parser.parse_args() stocks = EXTENDED_STOCKS if args.extended else DEFAULT_STOCKS tag = "12-stock" if args.extended else "5-stock" suffix = "_12stock" if args.extended else "" print(f"\n=== C30: Sector peer return features [{tag}] ===") print(f" New features: {C30_FEATURES}\n") # Load all 12 stocks for peer close prices print(" Loading all stock data for peer returns...") all_dfs: dict[str, pd.DataFrame] = {} for s in ALL_STOCKS: df = fetch_df(s) if df is not None and not df.empty: all_dfs[s] = df peer_close = _build_peer_close(all_dfs) print(f" Loaded {len(peer_close)} stocks\n") def _avg(results, key): vals = [m[key] for m in results if m and not np.isnan(m.get(key, float("nan")))] return round(float(np.mean(vals)), 1) if vals else float("nan") per_stock = {} base_results, c30_results = [], [] for stock_no in stocks: print(f" {stock_no}...", end=" ", flush=True) df = all_dfs.get(stock_no) if df is None or df.empty: print("no data"); continue feat = _build_features(df) feat = _add_peer_features(feat, df, stock_no, peer_close) close = (df.set_index("date")["close"] if "date" in df.columns else df["close"]).values labels = build_triple_barrier_labels(close) sector = STOCK_SECTOR.get(stock_no, "?") peers = [p for p in SECTOR_MAP.get(sector, []) if p != stock_no and p in peer_close] m_base = walk_forward(feat, labels, CURRENT_FEATURES) m_c30 = walk_forward(feat, labels, NEW_FEATURES) per_stock[stock_no] = {"baseline": m_base, "c30": m_c30, "peers": peers} base_results.append(m_base) c30_results.append(m_c30) b_dir = m_base.get("dir_accuracy", float("nan")) b_up = m_base.get("up_precision", float("nan")) c_dir = m_c30.get("dir_accuracy", float("nan")) c_up = m_c30.get("up_precision", float("nan")) print(f"peers={peers} base dir={b_dir}% ↑prec={b_up}% → c30 dir={c_dir}% ↑prec={c_up}%") base_agg = {"dir_accuracy": _avg(base_results, "dir_accuracy"), "up_precision": _avg(base_results, "up_precision")} c30_agg = {"dir_accuracy": _avg(c30_results, "dir_accuracy"), "up_precision": _avg(c30_results, "up_precision")} passed = c30_agg["dir_accuracy"] >= PASS_DIR_ACC and c30_agg["up_precision"] >= PASS_UP_PREC print(f"\n Baseline avg: dir={base_agg['dir_accuracy']}% ↑prec={base_agg['up_precision']}%") print(f" C30 avg: dir={c30_agg['dir_accuracy']}% ↑prec={c30_agg['up_precision']}%") print(f" Gate (dir≥{PASS_DIR_ACC}% AND ↑prec≥{PASS_UP_PREC}%): {'PASS ✓' if passed else 'FAIL ✗'}") result = { "experiment": "C30", "description": "Sector peer return features (peer_ret_1d, peer_ret_5d)", "new_features": C30_FEATURES, "stocks": stocks, "aggregate": {"baseline": base_agg, "c30": c30_agg}, "passed": passed, "pass_gate": {"dir_accuracy": PASS_DIR_ACC, "up_precision": PASS_UP_PREC}, "per_stock": per_stock, } out = ROOT / f"docs/c30_result{suffix}.json" out.parent.mkdir(exist_ok=True) out.write_text(json.dumps(result, indent=2)) print(f"\n Saved: {out}") return 0 if passed else 1 if __name__ == "__main__": sys.exit(main())