meta / data /loader.py
BharathPESU's picture
Upload data/loader.py with huggingface_hub
bc2dac3 verified
Raw History Blame Contribute Delete
2.32 kB
"""Price bar loading, validation, caching, and an optional yfinance fetcher.
Cache layout: <cache_dir>/<interval>/<SYMBOL>.parquet
Index: tz-aware timestamps in Asia/Kolkata. Columns: open, high, low, close, volume.
yfinance caveats: unofficial API, intraday history limited to recent days
(about 60 days for 5m bars). Use it to prototype, not to validate.
"""
from __future__ import annotations
from pathlib import Path
import pandas as pd
TZ = "Asia/Kolkata"
REQUIRED_COLUMNS = ["open", "high", "low", "close", "volume"]
def cache_path(cache_dir: Path | str, symbol: str, interval: str) -> Path:
return Path(cache_dir) / interval / f"{symbol}.parquet"
def validate_bars(df: pd.DataFrame) -> pd.DataFrame:
missing = [c for c in REQUIRED_COLUMNS if c not in df.columns]
if missing:
raise ValueError(f"bars missing columns: {missing}")
if df.index.tz is None:
raise ValueError("bar index must be timezone-aware (Asia/Kolkata)")
out = df.tz_convert(TZ).sort_index()
if out.index.duplicated().any():
raise ValueError("bar index has duplicate timestamps")
return out[REQUIRED_COLUMNS].astype(float)
def load_bars(symbol: str, cache_dir: Path | str, interval: str = "5min") -> pd.DataFrame | None:
path = cache_path(cache_dir, symbol, interval)
if not path.exists():
return None
return validate_bars(pd.read_parquet(path))
def save_bars(df: pd.DataFrame, symbol: str, cache_dir: Path | str, interval: str = "5min") -> Path:
path = cache_path(cache_dir, symbol, interval)
path.parent.mkdir(parents=True, exist_ok=True)
validate_bars(df).to_parquet(path)
return path
def fetch_yfinance(symbol: str, yf_interval: str = "5m", period: str = "60d") -> pd.DataFrame:
"""Download recent bars for an NSE symbol. TODO(verify) limits and data quality."""
import yfinance as yf
raw = yf.download(f"{symbol}.NS", interval=yf_interval, period=period,
auto_adjust=False, progress=False)
if raw.empty:
raise ValueError(f"no data returned for {symbol}")
if isinstance(raw.columns, pd.MultiIndex):
raw.columns = raw.columns.get_level_values(0)
df = raw.rename(columns=str.lower)
if df.index.tz is None:
df.index = df.index.tz_localize(TZ)
return validate_bars(df)