Spaces:
Sleeping
Sleeping
Khanna, Videh Rakesh Rakesh commited on
Commit Β·
f1c3844
1
Parent(s): 09c479e
fix: validation OHLCV fallback prefers fresher live data over stale SQL cache
Browse filesRoot cause: fetch_ohlcv()'s freshness gate rejected a source's data purely for
not being as fresh as 'today', even when the caller (validation's backdated
window fetch) only needed historical bars covering a past target date. This
threw away good data (e.g. jugaad lagging 'today' by a day) and fell back to
a multi-days-older SQL cache, so the window never covered the target date and
validation stayed stuck at 0 with an ever-growing overdue backlog.
Fix: track the most-recent non-fresh live result during the source loop and
prefer it over the SQL cache in the final fallback step when it's newer.
No signature change.
- data_sources.py +24 -1
data_sources.py
CHANGED
|
@@ -723,6 +723,12 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
|
|
| 723 |
]
|
| 724 |
|
| 725 |
# ββ 2b. Try each source ββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 726 |
for fn in sources:
|
| 727 |
try:
|
| 728 |
result = fn(ticker_ns, period)
|
|
@@ -730,10 +736,14 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
|
|
| 730 |
sc, sh, sl, sv = result
|
| 731 |
if ticker_ns in sc.columns and not sc[ticker_ns].dropna().empty:
|
| 732 |
if not _is_data_fresh(sc, ticker_ns):
|
|
|
|
| 733 |
logging.warning(
|
| 734 |
"OHLCV source %s returned stale data for %s (last: %s), trying next",
|
| 735 |
-
fn.__name__, ticker_ns,
|
| 736 |
)
|
|
|
|
|
|
|
|
|
|
| 737 |
continue
|
| 738 |
sc, sh, sl, sv = sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill()
|
| 739 |
_save_sql_cache(ticker_ns, period, sc, sh, sl, sv)
|
|
@@ -769,6 +779,19 @@ def fetch_ohlcv(ticker_ns: str, period: str = "1y"):
|
|
| 769 |
# threads (1D, 3D, 5D all queued behind 5D that just exhausted all sources)
|
| 770 |
# skip the full 73s chain and reach stale-cache / error in <1s.
|
| 771 |
_ohlcv_mark_failed(ticker_ns, period)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 772 |
if stale_cached is not None:
|
| 773 |
logging.warning(
|
| 774 |
"All live sources failed for %s β serving stale cached OHLCV", ticker_ns,
|
|
|
|
| 723 |
]
|
| 724 |
|
| 725 |
# ββ 2b. Try each source ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 726 |
+
# A source can return correct data that just isn't "fresh" (doesn't reach
|
| 727 |
+
# yesterday's bar yet, e.g. jugaad often lags a day) β that's still far
|
| 728 |
+
# better than a multi-day-old SQL cache for callers validating a backdated
|
| 729 |
+
# window. Remember the most-recent non-fresh result seen so 2d can prefer
|
| 730 |
+
# it over stale_cached instead of raising / serving even-older data.
|
| 731 |
+
best_live, best_live_last_date = None, None
|
| 732 |
for fn in sources:
|
| 733 |
try:
|
| 734 |
result = fn(ticker_ns, period)
|
|
|
|
| 736 |
sc, sh, sl, sv = result
|
| 737 |
if ticker_ns in sc.columns and not sc[ticker_ns].dropna().empty:
|
| 738 |
if not _is_data_fresh(sc, ticker_ns):
|
| 739 |
+
last_date = str(sc.index[-1])[:10]
|
| 740 |
logging.warning(
|
| 741 |
"OHLCV source %s returned stale data for %s (last: %s), trying next",
|
| 742 |
+
fn.__name__, ticker_ns, last_date,
|
| 743 |
)
|
| 744 |
+
if best_live_last_date is None or last_date > best_live_last_date:
|
| 745 |
+
best_live_last_date = last_date
|
| 746 |
+
best_live = (sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill())
|
| 747 |
continue
|
| 748 |
sc, sh, sl, sv = sc.ffill(), sh.ffill(), sl.ffill(), sv.ffill()
|
| 749 |
_save_sql_cache(ticker_ns, period, sc, sh, sl, sv)
|
|
|
|
| 779 |
# threads (1D, 3D, 5D all queued behind 5D that just exhausted all sources)
|
| 780 |
# skip the full 73s chain and reach stale-cache / error in <1s.
|
| 781 |
_ohlcv_mark_failed(ticker_ns, period)
|
| 782 |
+
|
| 783 |
+
# Prefer a live source's non-fresh-but-recent data over the SQL cache if it's
|
| 784 |
+
# newer (fixes backdated validation returning nothing when e.g. jugaad has the
|
| 785 |
+
# target date but lags "today" by a day, while the SQL cache predates it further).
|
| 786 |
+
stale_cache_last_date = str(stale_cached[0].index[-1])[:10] if stale_cached is not None else None
|
| 787 |
+
if best_live is not None and (stale_cache_last_date is None or best_live_last_date > stale_cache_last_date):
|
| 788 |
+
_save_sql_cache(ticker_ns, period, *best_live)
|
| 789 |
+
logging.warning(
|
| 790 |
+
"Using non-fresh live OHLCV for %s (last: %s) β newer than SQL cache (last: %s)",
|
| 791 |
+
ticker_ns, best_live_last_date, stale_cache_last_date,
|
| 792 |
+
)
|
| 793 |
+
return best_live
|
| 794 |
+
|
| 795 |
if stale_cached is not None:
|
| 796 |
logging.warning(
|
| 797 |
"All live sources failed for %s β serving stale cached OHLCV", ticker_ns,
|