Khanna, Videh Rakesh Rakesh commited on
Commit
f1c3844
Β·
1 Parent(s): 09c479e

fix: validation OHLCV fallback prefers fresher live data over stale SQL cache

Browse files

Root cause: fetch_ohlcv()'s freshness gate rejected a source's data purely for
not being as fresh as 'today', even when the caller (validation's backdated
window fetch) only needed historical bars covering a past target date. This
threw away good data (e.g. jugaad lagging 'today' by a day) and fell back to
a multi-days-older SQL cache, so the window never covered the target date and
validation stayed stuck at 0 with an ever-growing overdue backlog.

Fix: track the most-recent non-fresh live result during the source loop and
prefer it over the SQL cache in the final fallback step when it's newer.
No signature change.

Files changed (1) hide show
  1. data_sources.py +24 -1
data_sources.py CHANGED
@@ -723,6 +723,12 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
723
  ]
724
 
725
  # ── 2b. Try each source ────────────────────────────────────────────────
 
 
 
 
 
 
726
  for fn in sources:
727
  try:
728
  result = fn(ticker_ns, period)
@@ -730,10 +736,14 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
730
  sc, sh, sl, sv = result
731
  if ticker_ns in sc.columns and not sc[ticker_ns].dropna().empty:
732
  if not _is_data_fresh(sc, ticker_ns):
 
733
  logging.warning(
734
  "OHLCV source %s returned stale data for %s (last: %s), trying next",
735
- fn.__name__, ticker_ns, str(sc.index[-1])[:10],
736
  )
 
 
 
737
  continue
738
  sc, sh, sl, sv = sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill()
739
  _save_sql_cache(ticker_ns, period, sc, sh, sl, sv)
@@ -769,6 +779,19 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
769
  # threads (1D, 3D, 5D all queued behind 5D that just exhausted all sources)
770
  # skip the full 73s chain and reach stale-cache / error in <1s.
771
  _ohlcv_mark_failed(ticker_ns, period)
 
 
 
 
 
 
 
 
 
 
 
 
 
772
  if stale_cached is not None:
773
  logging.warning(
774
  "All live sources failed for %s β€” serving stale cached OHLCV", ticker_ns,
 
723
  ]
724
 
725
  # ── 2b. Try each source ────────────────────────────────────────────────
726
+ # A source can return correct data that just isn't "fresh" (doesn't reach
727
+ # yesterday's bar yet, e.g. jugaad often lags a day) β€” that's still far
728
+ # better than a multi-day-old SQL cache for callers validating a backdated
729
+ # window. Remember the most-recent non-fresh result seen so 2d can prefer
730
+ # it over stale_cached instead of raising / serving even-older data.
731
+ best_live, best_live_last_date = None, None
732
  for fn in sources:
733
  try:
734
  result = fn(ticker_ns, period)
 
736
  sc, sh, sl, sv = result
737
  if ticker_ns in sc.columns and not sc[ticker_ns].dropna().empty:
738
  if not _is_data_fresh(sc, ticker_ns):
739
+ last_date = str(sc.index[-1])[:10]
740
  logging.warning(
741
  "OHLCV source %s returned stale data for %s (last: %s), trying next",
742
+ fn.__name__, ticker_ns, last_date,
743
  )
744
+ if best_live_last_date is None or last_date > best_live_last_date:
745
+ best_live_last_date = last_date
746
+ best_live = (sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill())
747
  continue
748
  sc, sh, sl, sv = sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill()
749
  _save_sql_cache(ticker_ns, period, sc, sh, sl, sv)
 
779
  # threads (1D, 3D, 5D all queued behind 5D that just exhausted all sources)
780
  # skip the full 73s chain and reach stale-cache / error in <1s.
781
  _ohlcv_mark_failed(ticker_ns, period)
782
+
783
+ # Prefer a live source's non-fresh-but-recent data over the SQL cache if it's
784
+ # newer (fixes backdated validation returning nothing when e.g. jugaad has the
785
+ # target date but lags "today" by a day, while the SQL cache predates it further).
786
+ stale_cache_last_date = str(stale_cached[0].index[-1])[:10] if stale_cached is not None else None
787
+ if best_live is not None and (stale_cache_last_date is None or best_live_last_date > stale_cache_last_date):
788
+ _save_sql_cache(ticker_ns, period, *best_live)
789
+ logging.warning(
790
+ "Using non-fresh live OHLCV for %s (last: %s) β€” newer than SQL cache (last: %s)",
791
+ ticker_ns, best_live_last_date, stale_cache_last_date,
792
+ )
793
+ return best_live
794
+
795
  if stale_cached is not None:
796
  logging.warning(
797
  "All live sources failed for %s β€” serving stale cached OHLCV", ticker_ns,