"""Walk-forward check: how good were next-day forecasts over recent sessions? The model is fitted once on data before the test window. Then, one day at a time, it forecasts the next close and only afterwards sees the real one. The parameters stay frozen, so no test data leaks into the fit. The yardstick is the naive forecast "tomorrow closes where today closed". Daily prices move little, so a low MAPE on its own says almost nothing. The question is whether the model beats that. """ from __future__ import annotations from dataclasses import dataclass import numpy as np import pandas as pd from .model import FIT_WINDOW, fit TEST_DAYS = 20 @dataclass(frozen=True) class Backtest: dates: pd.DatetimeIndex actual: np.ndarray model: np.ndarray naive: np.ndarray model_mape: float naive_mape: float model_mae: float naive_mae: float direction_hit: float # share of days where the model got up/down right direction_days: int def _mape(actual: np.ndarray, pred: np.ndarray) -> float: return float(np.mean(np.abs((actual - pred) / actual))) def run(closes: pd.Series, test_days: int = TEST_DAYS) -> Backtest: series = closes.iloc[-(FIT_WINDOW + test_days):] values = series.to_numpy(dtype=float) if len(values) < test_days + 60: raise ValueError("not enough history for a backtest") train, test = values[:-test_days], values[-test_days:] result = fit(train) preds = np.empty(test_days) for i, actual in enumerate(test): preds[i] = float(np.asarray(result.forecast(steps=1))[0]) result = result.append([actual]) # reveal the real close, keep params naive = np.concatenate([[train[-1]], test[:-1]]) moved = np.sign(test - naive) called = np.sign(preds - naive) mask = moved != 0 # flat days have no direction to call hit = float(np.mean(moved[mask] == called[mask])) if mask.any() else float("nan") return Backtest( dates=series.index[-test_days:], actual=test, model=preds, naive=naive, model_mape=_mape(test, preds), naive_mape=_mape(test, naive), model_mae=float(np.mean(np.abs(test - preds))), naive_mae=float(np.mean(np.abs(test - naive))), direction_hit=hit, direction_days=int(mask.sum()), )