"""Split/dividend adjustment for a user's raw 1-minute OHLCV bars. Reproduces the convention the model was trained under (ported from the authors' training pipeline): * factors are cumulative and backward-looking, anchored at 1.0 on the latest bar date; * a split with ex-date D multiplies price by denominator/numerator and volume by numerator/denominator on every bar dated strictly before D (Eastern time); * a cash dividend with ex-date D multiplies price by (close_prior - amount) / close_prior on every bar dated strictly before D, where close_prior is the close of the last bar stamped at or before 16:00 ET on the last bar date before D (or that day's last bar if none); volume is unchanged; * dividends with amount >= close_prior are skipped (price left under-adjusted); * two actions on the same ex-date are combined: amounts summed, split ratios multiplied. Only pandas and numpy are required. """ from __future__ import annotations import datetime as dt import numpy as np import pandas as pd EASTERN = "America/New_York" REGULAR_CLOSE = dt.time(16, 0) def clean_bars(bars: pd.DataFrame) -> pd.DataFrame: """Apply the training pipeline's raw-bar quality checks. Drops duplicate timestamps (keeps first), bars with high < low, bars whose close lies outside [low, high], and bars with a non-positive or missing price; negative or missing volume is set to 0. """ df = bars.drop_duplicates(subset="timestamp", keep="first") ok = (df["high"] >= df["low"]) & (df["close"] <= df["high"]) & (df["close"] >= df["low"]) ok &= (df[["open", "high", "low", "close"]] > 0).all(axis=1) df = df[ok.fillna(False)].copy() df["volume"] = df["volume"].fillna(0).clip(lower=0) return df.sort_values("timestamp").reset_index(drop=True) def _parse_timestamps(ts: pd.Series) -> pd.Series: """Parse strings; mixed UTC offsets (e.g. across a DST change) are parsed as UTC. pandas 3 raises on mixed offsets, pandas 2 returns object dtype; both fall back. """ try: parsed = pd.to_datetime(ts) except ValueError: return pd.to_datetime(ts, utc=True) if not pd.api.types.is_datetime64_any_dtype(parsed): return pd.to_datetime(ts, utc=True) return parsed def _eastern(ts: pd.Series) -> pd.Series: """Parse if needed, then localize naive as Eastern or convert aware to Eastern.""" if not pd.api.types.is_datetime64_any_dtype(ts): ts = _parse_timestamps(ts) return ts.dt.tz_localize(EASTERN) if ts.dt.tz is None else ts.dt.tz_convert(EASTERN) def _split_events(splits: pd.DataFrame | None) -> dict[dt.date, tuple[float, float]]: """{ex_date: (numerator, denominator)}; accepts `ratio` (10.0 = 10-for-1, 0.5 = 1-for-2). Same-date splits are multiplied together. """ if splits is None or len(splits) == 0: return {} s = splits.copy() if "numerator" not in s: s["numerator"], s["denominator"] = s["ratio"].astype(float), 1.0 events: dict[dt.date, tuple[float, float]] = {} for d, n, m in zip(s["date"], s["numerator"], s["denominator"], strict=True): ex_date = pd.Timestamp(d).date() n0, m0 = events.get(ex_date, (1.0, 1.0)) events[ex_date] = (n0 * float(n), m0 * float(m)) return events def _close_prior(df: pd.DataFrame, ex_date: dt.date) -> float | None: before = df[df["_date"] < ex_date] if before.empty: return None last_day = before[before["_date"] == before["_date"].iloc[-1]] regular = last_day[last_day["timestamp_eastern"].dt.time <= REGULAR_CLOSE] return float((regular if not regular.empty else last_day)["close"].iloc[-1]) def _add_dividend(events: dict, df: pd.DataFrame, ex_date: dt.date, amount: float, close_prior) -> None: """Insert one dividend; a repeat ex-date adds to the amount already recorded.""" if ex_date in events: events[ex_date] = (events[ex_date][0] + amount, events[ex_date][1]) return if pd.isna(close_prior): close_prior = _close_prior(df, ex_date) if close_prior is not None and close_prior > 0: events[ex_date] = (amount, float(close_prior)) def _dividend_events(df: pd.DataFrame, dividends: pd.DataFrame | None) -> dict[dt.date, tuple[float, float]]: """{ex_date: (amount, close_prior)}; close_prior is taken from the bars unless given. Same-date dividends are summed against one close_prior. """ if dividends is None or len(dividends) == 0: return {} events: dict[dt.date, tuple[float, float]] = {} given = dividends["close_prior"] if "close_prior" in dividends else [None] * len(dividends) for d, amount, cp in zip(dividends["date"], dividends["amount"], given, strict=True): _add_dividend(events, df, pd.Timestamp(d).date(), float(amount), cp) return events def _apply_event(price: float, volume: float, ev: dt.date, splits: dict, divs: dict) -> tuple[float, float]: """Fold one ex-date's split and/or dividend into the running factors.""" if ev in splits: num, den = splits[ev] price *= den / num volume *= num / den if ev in divs: amount, close_prior = divs[ev] if close_prior > amount: price *= (close_prior - amount) / close_prior return price, volume def _factors_by_date(dates: list[dt.date], splits: dict, divs: dict) -> tuple[dict, dict]: """Cumulative backward-looking factors per bar date, anchored at 1.0 on the latest date. Walking backwards, every event with `current < ex-date <= previous date` scales all bars dated `current` and earlier. """ event_dates = sorted(set(splits) | set(divs)) price, volume = 1.0, 1.0 price_by, volume_by = {}, {} prev = None for current in reversed(dates): for ev in event_dates: if prev is not None and current < ev <= prev: price, volume = _apply_event(price, volume, ev, splits, divs) price_by[current], volume_by[current] = price, volume prev = current return price_by, volume_by def adjust_ohlcv( bars: pd.DataFrame, splits: pd.DataFrame | None = None, dividends: pd.DataFrame | None = None, ) -> pd.DataFrame: """Return cleaned bars with adjusted prices ready for `make_features`. Args: bars: one security; columns `timestamp` (bar start; tz-aware, or naive taken as Eastern), `open`, `high`, `low`, `close`, `volume`, all unadjusted. splits: columns `date` (ex-date) and either `ratio` or `numerator`+`denominator` (numerator = shares after the split). dividends: columns `date` (ex-date), `amount` (cash per share as declared), optional `close_prior`. Returns: DataFrame with `timestamp_eastern`, `open_adj`, `high_adj`, `low_adj`, `close_adj`, `volume_adj`, `adj_factor_price`, `adj_factor_volume`, sorted by time. """ df = clean_bars(bars) df["timestamp_eastern"] = _eastern(df["timestamp"]) df["_date"] = df["timestamp_eastern"].dt.date price_by, volume_by = _factors_by_date( sorted(df["_date"].unique()), _split_events(splits), _dividend_events(df, dividends) ) df["adj_factor_price"] = df["_date"].map(price_by).astype(np.float64) df["adj_factor_volume"] = df["_date"].map(volume_by).astype(np.float64) for col in ("open", "high", "low", "close"): df[f"{col}_adj"] = df[col] * df["adj_factor_price"] df["volume_adj"] = df["volume"] * df["adj_factor_volume"] keep = [ "timestamp_eastern", "open_adj", "high_adj", "low_adj", "close_adj", "volume_adj", "adj_factor_price", "adj_factor_volume", ] return df[keep]