tests/test_data.py

"""Phase 0 acceptance: provider chain, network gate, incremental refresh.

No test here touches the network; provider fetchers are substituted.
"""

from __future__ import annotations

import time

import pandas as pd
import pytest

from src import config, data
from src.data import (
    NetworkNotAllowed,
    ProviderError,
    RateLimiter,
    allow_network,
    fetch_ohlcv,
    missing_price_ranges,
    refresh,
)
from src.store import SignalStore


@pytest.fixture
def store(tmp_path) -> SignalStore:
    return SignalStore(repo_id=None, local_root=tmp_path / "store", offline=True)


def ohlcv(n=30, start="2024-01-01", freq="D", source="fake", base=100.0) -> pd.DataFrame:
    ts = pd.date_range(start, periods=n, freq=freq, tz="UTC")
    close = pd.Series([base + i for i in range(n)], dtype="float64")
    return pd.DataFrame({
        "ts": ts, "open": close * 0.99, "high": close * 1.02,
        "low": close * 0.98, "close": close, "volume": 1000.0, "source": source,
    })


@pytest.fixture
def fake_chain(monkeypatch):
    """Replace every provider fetcher with a recorder we control."""
    calls: list[str] = []

    def make(name, *, rows=30, fail=False, start="2024-01-01"):
        def _f(spec, asset, tf, s, e):
            calls.append(name)
            if fail:
                raise ProviderError(f"{name} is down")
            return ohlcv(n=rows, start=start, source=name)
        return _f

    monkeypatch.setattr(data, "_FETCHERS", {}, raising=False)
    return calls, make


# --------------------------------------------------------------------------
# Network gate — user-facing paths must never reach a provider
# --------------------------------------------------------------------------


def test_fetch_is_blocked_outside_the_refresh_path():
    with pytest.raises(NetworkNotAllowed, match="cached store"):
        fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01")


def test_gate_closes_again_after_the_context_exits():
    assert not data.network_allowed()
    with allow_network():
        assert data.network_allowed()
    assert not data.network_allowed()


def test_gate_is_thread_local():
    import threading

    seen = {}

    def worker():
        seen["other"] = data.network_allowed()

    with allow_network():
        t = threading.Thread(target=worker)
        t.start()
        t.join()
    assert seen["other"] is False


# --------------------------------------------------------------------------
# Chain walk / fallback
# --------------------------------------------------------------------------


def test_primary_provider_wins(fake_chain, monkeypatch):
    calls, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance"))
    monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase"))
    with allow_network():
        res = fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01")
    assert res.source == "binance"
    assert calls == ["binance"]


def test_falls_back_when_primary_fails(fake_chain, monkeypatch):
    calls, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True))
    monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase"))
    with allow_network():
        res = fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01")
    assert res.source == "coinbase"
    assert calls == ["binance", "coinbase"]


def test_exhausted_chain_raises_with_every_reason(fake_chain, monkeypatch):
    _, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True))
    monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase", fail=True))
    with allow_network():
        with pytest.raises(ProviderError) as exc:
            fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01")
    assert "binance is down" in str(exc.value)
    assert "coinbase is down" in str(exc.value)


def test_equity_and_crypto_use_different_chains():
    crypto = [p.name for p in config.providers_for("crypto")]
    equity = [p.name for p in config.providers_for("equity")]
    assert crypto[:2] == ["binance", "coinbase"]
    assert equity[:2] == ["yfinance", "stooq"]
    assert not set(crypto) & set(equity)


def test_tiingo_only_joins_the_chain_when_its_key_is_present(monkeypatch):
    monkeypatch.delenv("TIINGO_KEY", raising=False)
    assert "tiingo" not in [p.name for p in config.providers_for("equity")]
    monkeypatch.setenv("TIINGO_KEY", "x")
    assert "tiingo" in [p.name for p in config.providers_for("equity")]


def test_unknown_asset_is_rejected():
    with allow_network():
        with pytest.raises(ProviderError, match="unknown asset"):
            fetch_ohlcv("DOGE-USD", "1d", "2024-01-01", "2024-02-01")


# --------------------------------------------------------------------------
# Rate limiting + backoff
# --------------------------------------------------------------------------


def test_rate_limiter_spaces_calls():
    name = "unit-test-limiter"
    RateLimiter.wait(name, 0.0)
    t0 = time.monotonic()
    RateLimiter.wait(name, 0.15)
    assert time.monotonic() - t0 >= 0.14


def test_backoff_retries_then_succeeds(monkeypatch):
    monkeypatch.setattr(time, "sleep", lambda s: None)
    spec = config.ProviderSpec("retry-test", ("crypto",), min_interval_s=0.0,
                               max_retries=4, backoff_base_s=1.0)
    attempts = {"n": 0}

    def flaky():
        attempts["n"] += 1
        if attempts["n"] < 3:
            raise RuntimeError("transient")
        return "ok"

    assert data.with_backoff(spec, flaky) == "ok"
    assert attempts["n"] == 3


def test_backoff_gives_up_and_reports(monkeypatch):
    monkeypatch.setattr(time, "sleep", lambda s: None)
    spec = config.ProviderSpec("giveup-test", ("crypto",), min_interval_s=0.0,
                               max_retries=3, backoff_base_s=1.0)

    def always_fails():
        raise RuntimeError("nope")

    with pytest.raises(ProviderError, match="exhausted retries"):
        data.with_backoff(spec, always_fails)


# --------------------------------------------------------------------------
# Incremental refresh — only fetch what is missing
# --------------------------------------------------------------------------


def test_missing_ranges_on_empty_cache(store):
    gaps = missing_price_ranges(store, "BTC-USD", "1d", "2024-01-01", "2024-03-01")
    assert len(gaps) == 1


def test_missing_ranges_fully_cached(store):
    store.write_prices("BTC-USD", "1d", ohlcv(n=60, start="2024-01-01"))
    assert missing_price_ranges(store, "BTC-USD", "1d",
                                "2024-01-10", "2024-02-10") == []


def test_missing_ranges_finds_both_edges(store):
    store.write_prices("BTC-USD", "1d", ohlcv(n=30, start="2024-02-01"))
    gaps = missing_price_ranges(store, "BTC-USD", "1d", "2024-01-01", "2024-04-01")
    assert len(gaps) == 2
    assert gaps[0][0] < pd.Timestamp("2024-02-01", tz="UTC")
    assert gaps[1][1] > pd.Timestamp("2024-03-01", tz="UTC")


def test_refresh_skips_a_fully_cached_range(store, fake_chain, monkeypatch):
    calls, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=60))
    store.write_prices("BTC-USD", "1d", ohlcv(n=60, start="2024-01-01"))

    rep = refresh(store, "BTC-USD", "1d", "2024-01-10", "2024-02-10")
    assert rep.skipped_cached and rep.rows_added == 0
    assert calls == []  # no provider was contacted


def test_refresh_fetches_and_writes(store, fake_chain, monkeypatch):
    calls, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=30))
    rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30")
    assert rep.ok and rep.rows_added == 30
    assert rep.sources == ["binance"]
    assert len(store.get_prices("BTC-USD", "1d")) == 30


def test_refresh_reports_failure_rather_than_returning_partial_data(
    store, fake_chain, monkeypatch
):
    _, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True))
    monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase", fail=True))
    rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30")
    assert not rep.ok
    assert rep.rows_added == 0
    assert "FAILED" in rep.summary()


def test_refresh_surfaces_dirty_data_instead_of_writing_it(
    store, fake_chain, monkeypatch
):
    def bad(spec, asset, tf, s, e):
        df = ohlcv(n=30, source="binance")
        df.loc[5, "close"] = -1.0
        return df

    monkeypatch.setitem(data._FETCHERS, "binance", bad)
    monkeypatch.setitem(data._FETCHERS, "coinbase", bad)
    rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30")
    assert not rep.ok
    assert any("non-positive" in e for e in rep.errors)
    assert store.get_prices("BTC-USD", "1d").empty


def test_refresh_records_source_per_row(store, fake_chain, monkeypatch):
    _, make = fake_chain
    monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=20))
    refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-20")
    got = store.get_prices("BTC-USD", "1d")
    assert set(got["source"].unique()) == {"binance"}


def test_provider_depth_limit_is_recorded_as_a_boundary_not_an_error(store):
    """Yahoo serves ~730d of 1h bars; that is coverage truth, not a failure."""
    store.write_prices("SPY", "1h", ohlcv(n=48, start="2025-01-01", freq="h"))
    cov = store.load_manifest().prices["SPY|1h"]
    assert cov.provider_max_days == config.TIMEFRAMES["1h"].yahoo_max_days == 730