Spaces:
Sleeping
Sleeping
| """Walk-forward check: how good were next-day forecasts over recent sessions? | |
| The model is fitted once on data before the test window. Then, one day at a | |
| time, it forecasts the next close and only afterwards sees the real one. The | |
| parameters stay frozen, so no test data leaks into the fit. | |
| The yardstick is the naive forecast "tomorrow closes where today closed". | |
| Daily prices move little, so a low MAPE on its own says almost nothing. The | |
| question is whether the model beats that. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import numpy as np | |
| import pandas as pd | |
| from .model import FIT_WINDOW, fit | |
| TEST_DAYS = 20 | |
| class Backtest: | |
| dates: pd.DatetimeIndex | |
| actual: np.ndarray | |
| model: np.ndarray | |
| naive: np.ndarray | |
| model_mape: float | |
| naive_mape: float | |
| model_mae: float | |
| naive_mae: float | |
| direction_hit: float # share of days where the model got up/down right | |
| direction_days: int | |
| def _mape(actual: np.ndarray, pred: np.ndarray) -> float: | |
| return float(np.mean(np.abs((actual - pred) / actual))) | |
| def run(closes: pd.Series, test_days: int = TEST_DAYS) -> Backtest: | |
| series = closes.iloc[-(FIT_WINDOW + test_days):] | |
| values = series.to_numpy(dtype=float) | |
| if len(values) < test_days + 60: | |
| raise ValueError("not enough history for a backtest") | |
| train, test = values[:-test_days], values[-test_days:] | |
| result = fit(train) | |
| preds = np.empty(test_days) | |
| for i, actual in enumerate(test): | |
| preds[i] = float(np.asarray(result.forecast(steps=1))[0]) | |
| result = result.append([actual]) # reveal the real close, keep params | |
| naive = np.concatenate([[train[-1]], test[:-1]]) | |
| moved = np.sign(test - naive) | |
| called = np.sign(preds - naive) | |
| mask = moved != 0 # flat days have no direction to call | |
| hit = float(np.mean(moved[mask] == called[mask])) if mask.any() else float("nan") | |
| return Backtest( | |
| dates=series.index[-test_days:], | |
| actual=test, | |
| model=preds, | |
| naive=naive, | |
| model_mape=_mape(test, preds), | |
| naive_mape=_mape(test, naive), | |
| model_mae=float(np.mean(np.abs(test - preds))), | |
| naive_mae=float(np.mean(np.abs(test - naive))), | |
| direction_hit=hit, | |
| direction_days=int(mask.sum()), | |
| ) | |