"""Phase 0 acceptance: provider chain, network gate, incremental refresh. No test here touches the network; provider fetchers are substituted. """ from __future__ import annotations import time import pandas as pd import pytest from src import config, data from src.data import ( NetworkNotAllowed, ProviderError, RateLimiter, allow_network, fetch_ohlcv, missing_price_ranges, refresh, ) from src.store import SignalStore @pytest.fixture def store(tmp_path) -> SignalStore: return SignalStore(repo_id=None, local_root=tmp_path / "store", offline=True) def ohlcv(n=30, start="2024-01-01", freq="D", source="fake", base=100.0) -> pd.DataFrame: ts = pd.date_range(start, periods=n, freq=freq, tz="UTC") close = pd.Series([base + i for i in range(n)], dtype="float64") return pd.DataFrame({ "ts": ts, "open": close * 0.99, "high": close * 1.02, "low": close * 0.98, "close": close, "volume": 1000.0, "source": source, }) @pytest.fixture def fake_chain(monkeypatch): """Replace every provider fetcher with a recorder we control.""" calls: list[str] = [] def make(name, *, rows=30, fail=False, start="2024-01-01"): def _f(spec, asset, tf, s, e): calls.append(name) if fail: raise ProviderError(f"{name} is down") return ohlcv(n=rows, start=start, source=name) return _f monkeypatch.setattr(data, "_FETCHERS", {}, raising=False) return calls, make # -------------------------------------------------------------------------- # Network gate — user-facing paths must never reach a provider # -------------------------------------------------------------------------- def test_fetch_is_blocked_outside_the_refresh_path(): with pytest.raises(NetworkNotAllowed, match="cached store"): fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01") def test_gate_closes_again_after_the_context_exits(): assert not data.network_allowed() with allow_network(): assert data.network_allowed() assert not data.network_allowed() def test_gate_is_thread_local(): import threading seen = {} def worker(): seen["other"] = data.network_allowed() with allow_network(): t = threading.Thread(target=worker) t.start() t.join() assert seen["other"] is False # -------------------------------------------------------------------------- # Chain walk / fallback # -------------------------------------------------------------------------- def test_primary_provider_wins(fake_chain, monkeypatch): calls, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance")) monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase")) with allow_network(): res = fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01") assert res.source == "binance" assert calls == ["binance"] def test_falls_back_when_primary_fails(fake_chain, monkeypatch): calls, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True)) monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase")) with allow_network(): res = fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01") assert res.source == "coinbase" assert calls == ["binance", "coinbase"] def test_exhausted_chain_raises_with_every_reason(fake_chain, monkeypatch): _, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True)) monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase", fail=True)) with allow_network(): with pytest.raises(ProviderError) as exc: fetch_ohlcv("BTC-USD", "1d", "2024-01-01", "2024-02-01") assert "binance is down" in str(exc.value) assert "coinbase is down" in str(exc.value) def test_equity_and_crypto_use_different_chains(): crypto = [p.name for p in config.providers_for("crypto")] equity = [p.name for p in config.providers_for("equity")] assert crypto[:2] == ["binance", "coinbase"] assert equity[:2] == ["yfinance", "stooq"] assert not set(crypto) & set(equity) def test_tiingo_only_joins_the_chain_when_its_key_is_present(monkeypatch): monkeypatch.delenv("TIINGO_KEY", raising=False) assert "tiingo" not in [p.name for p in config.providers_for("equity")] monkeypatch.setenv("TIINGO_KEY", "x") assert "tiingo" in [p.name for p in config.providers_for("equity")] def test_unknown_asset_is_rejected(): with allow_network(): with pytest.raises(ProviderError, match="unknown asset"): fetch_ohlcv("DOGE-USD", "1d", "2024-01-01", "2024-02-01") # -------------------------------------------------------------------------- # Rate limiting + backoff # -------------------------------------------------------------------------- def test_rate_limiter_spaces_calls(): name = "unit-test-limiter" RateLimiter.wait(name, 0.0) t0 = time.monotonic() RateLimiter.wait(name, 0.15) assert time.monotonic() - t0 >= 0.14 def test_backoff_retries_then_succeeds(monkeypatch): monkeypatch.setattr(time, "sleep", lambda s: None) spec = config.ProviderSpec("retry-test", ("crypto",), min_interval_s=0.0, max_retries=4, backoff_base_s=1.0) attempts = {"n": 0} def flaky(): attempts["n"] += 1 if attempts["n"] < 3: raise RuntimeError("transient") return "ok" assert data.with_backoff(spec, flaky) == "ok" assert attempts["n"] == 3 def test_backoff_gives_up_and_reports(monkeypatch): monkeypatch.setattr(time, "sleep", lambda s: None) spec = config.ProviderSpec("giveup-test", ("crypto",), min_interval_s=0.0, max_retries=3, backoff_base_s=1.0) def always_fails(): raise RuntimeError("nope") with pytest.raises(ProviderError, match="exhausted retries"): data.with_backoff(spec, always_fails) # -------------------------------------------------------------------------- # Incremental refresh — only fetch what is missing # -------------------------------------------------------------------------- def test_missing_ranges_on_empty_cache(store): gaps = missing_price_ranges(store, "BTC-USD", "1d", "2024-01-01", "2024-03-01") assert len(gaps) == 1 def test_missing_ranges_fully_cached(store): store.write_prices("BTC-USD", "1d", ohlcv(n=60, start="2024-01-01")) assert missing_price_ranges(store, "BTC-USD", "1d", "2024-01-10", "2024-02-10") == [] def test_missing_ranges_finds_both_edges(store): store.write_prices("BTC-USD", "1d", ohlcv(n=30, start="2024-02-01")) gaps = missing_price_ranges(store, "BTC-USD", "1d", "2024-01-01", "2024-04-01") assert len(gaps) == 2 assert gaps[0][0] < pd.Timestamp("2024-02-01", tz="UTC") assert gaps[1][1] > pd.Timestamp("2024-03-01", tz="UTC") def test_refresh_skips_a_fully_cached_range(store, fake_chain, monkeypatch): calls, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=60)) store.write_prices("BTC-USD", "1d", ohlcv(n=60, start="2024-01-01")) rep = refresh(store, "BTC-USD", "1d", "2024-01-10", "2024-02-10") assert rep.skipped_cached and rep.rows_added == 0 assert calls == [] # no provider was contacted def test_refresh_fetches_and_writes(store, fake_chain, monkeypatch): calls, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=30)) rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30") assert rep.ok and rep.rows_added == 30 assert rep.sources == ["binance"] assert len(store.get_prices("BTC-USD", "1d")) == 30 def test_refresh_reports_failure_rather_than_returning_partial_data( store, fake_chain, monkeypatch ): _, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", fail=True)) monkeypatch.setitem(data._FETCHERS, "coinbase", make("coinbase", fail=True)) rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30") assert not rep.ok assert rep.rows_added == 0 assert "FAILED" in rep.summary() def test_refresh_surfaces_dirty_data_instead_of_writing_it( store, fake_chain, monkeypatch ): def bad(spec, asset, tf, s, e): df = ohlcv(n=30, source="binance") df.loc[5, "close"] = -1.0 return df monkeypatch.setitem(data._FETCHERS, "binance", bad) monkeypatch.setitem(data._FETCHERS, "coinbase", bad) rep = refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-30") assert not rep.ok assert any("non-positive" in e for e in rep.errors) assert store.get_prices("BTC-USD", "1d").empty def test_refresh_records_source_per_row(store, fake_chain, monkeypatch): _, make = fake_chain monkeypatch.setitem(data._FETCHERS, "binance", make("binance", rows=20)) refresh(store, "BTC-USD", "1d", "2024-01-01", "2024-01-20") got = store.get_prices("BTC-USD", "1d") assert set(got["source"].unique()) == {"binance"} def test_provider_depth_limit_is_recorded_as_a_boundary_not_an_error(store): """Yahoo serves ~730d of 1h bars; that is coverage truth, not a failure.""" store.write_prices("SPY", "1h", ohlcv(n=48, start="2025-01-01", freq="h")) cov = store.load_manifest().prices["SPY|1h"] assert cov.provider_max_days == config.TIMEFRAMES["1h"].yahoo_max_days == 730