Spaces:
Runtime error
Runtime error
Update data_us.py
Browse files- data_us.py +30 -6
data_us.py
CHANGED
|
@@ -4,7 +4,7 @@ data_us.py — US market data layer (yfinance), replacing baostock/pytdx.
|
|
| 4 |
Levels & history limits (Yahoo Finance API constraints):
|
| 5 |
daily : 10 years (weekly / monthly are resampled from daily
|
| 6 |
by chan_multilevel.resample_weekly/_monthly)
|
| 7 |
-
60m : last 730 days
|
| 8 |
30m/15m : last 60 days
|
| 9 |
5m : last 60 days
|
| 10 |
1m : last 7 days only → too short for Chan decomposition, NOT used.
|
|
@@ -20,6 +20,7 @@ and refreshed when stale (daily: >12h old, intraday: >2h old) or on force=True.
|
|
| 20 |
from __future__ import annotations
|
| 21 |
|
| 22 |
import os
|
|
|
|
| 23 |
import time
|
| 24 |
import traceback
|
| 25 |
|
|
@@ -27,12 +28,22 @@ import pandas as pd
|
|
| 27 |
|
| 28 |
import paths
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
CACHE_DIR = os.environ.get("CHAN_CACHE_DIR", paths.CACHE_DIR)
|
| 31 |
|
| 32 |
LEVELS = {
|
| 33 |
-
# level: (yfinance interval,
|
|
|
|
|
|
|
| 34 |
"d": ("1d", "10y"),
|
| 35 |
-
"60m": ("1h",
|
| 36 |
"30m": ("30m", "60d"),
|
| 37 |
"15m": ("15m", "60d"),
|
| 38 |
"5m": ("5m", "60d"),
|
|
@@ -91,9 +102,22 @@ def load_level(ticker: str, level: str, force: bool = False) -> pd.DataFrame:
|
|
| 91 |
pass
|
| 92 |
try:
|
| 93 |
import yfinance as yf
|
|
|
|
| 94 |
interval, period = LEVELS[level]
|
| 95 |
-
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
df = _normalize(raw)
|
| 98 |
if len(df):
|
| 99 |
df.to_parquet(path, index=False)
|
|
@@ -144,4 +168,4 @@ def prefetch(tickers, levels=FAST_LEVELS, force: bool = False, workers: int = 5,
|
|
| 144 |
|
| 145 |
def last_daily_date(ticker: str):
|
| 146 |
df = load_level(ticker, "d")
|
| 147 |
-
return None if df.empty else pd.Timestamp(df["date"].iloc[-1])
|
|
|
|
| 4 |
Levels & history limits (Yahoo Finance API constraints):
|
| 5 |
daily : 10 years (weekly / monthly are resampled from daily
|
| 6 |
by chan_multilevel.resample_weekly/_monthly)
|
| 7 |
+
60m : last 730 days (fetched as "1h" interval with explicit start/end dates)
|
| 8 |
30m/15m : last 60 days
|
| 9 |
5m : last 60 days
|
| 10 |
1m : last 7 days only → too short for Chan decomposition, NOT used.
|
|
|
|
| 20 |
from __future__ import annotations
|
| 21 |
|
| 22 |
import os
|
| 23 |
+
import threading
|
| 24 |
import time
|
| 25 |
import traceback
|
| 26 |
|
|
|
|
| 28 |
|
| 29 |
import paths
|
| 30 |
|
| 31 |
+
# yfinance uses a shared SQLite cache (peewee) for timezone lookups.
|
| 32 |
+
# When multiple threads call yf.Ticker().history() simultaneously the DB
|
| 33 |
+
# gets locked and raises peewee.OperationalError, stalling prefetch and
|
| 34 |
+
# freezing the "Run analysis" button. Serialise the yfinance connect/lookup
|
| 35 |
+
# phase with a process-wide lock — Yahoo's own rate-limit is the real
|
| 36 |
+
# bottleneck anyway, so the extra serialisation costs almost nothing.
|
| 37 |
+
_YF_LOCK = threading.Lock()
|
| 38 |
+
|
| 39 |
CACHE_DIR = os.environ.get("CHAN_CACHE_DIR", paths.CACHE_DIR)
|
| 40 |
|
| 41 |
LEVELS = {
|
| 42 |
+
# level: (yfinance interval, period_or_days)
|
| 43 |
+
# For "60m" Yahoo requires explicit start/end dates (not a period string)
|
| 44 |
+
# when fetching more than ~60 days back; we pass days as an int sentinel.
|
| 45 |
"d": ("1d", "10y"),
|
| 46 |
+
"60m": ("1h", "730d"), # use explicit start/end — "period='730d'" is rejected by Yahoo for 1h
|
| 47 |
"30m": ("30m", "60d"),
|
| 48 |
"15m": ("15m", "60d"),
|
| 49 |
"5m": ("5m", "60d"),
|
|
|
|
| 102 |
pass
|
| 103 |
try:
|
| 104 |
import yfinance as yf
|
| 105 |
+
from datetime import datetime, timedelta
|
| 106 |
interval, period = LEVELS[level]
|
| 107 |
+
# Acquire lock before any yfinance call — the shared peewee/SQLite
|
| 108 |
+
# timezone cache raises "database is locked" under concurrent access.
|
| 109 |
+
with _YF_LOCK:
|
| 110 |
+
if isinstance(period, int):
|
| 111 |
+
# Yahoo rejects period strings for hourly data older than ~60 days.
|
| 112 |
+
# Use explicit start/end timestamps instead.
|
| 113 |
+
end_dt = datetime.utcnow()
|
| 114 |
+
start_dt = end_dt - timedelta(days=period)
|
| 115 |
+
raw = yf.Ticker(ticker).history(start=start_dt, end=end_dt,
|
| 116 |
+
interval=interval,
|
| 117 |
+
auto_adjust=True, actions=False)
|
| 118 |
+
else:
|
| 119 |
+
raw = yf.Ticker(ticker).history(period=period, interval=interval,
|
| 120 |
+
auto_adjust=True, actions=False)
|
| 121 |
df = _normalize(raw)
|
| 122 |
if len(df):
|
| 123 |
df.to_parquet(path, index=False)
|
|
|
|
| 168 |
|
| 169 |
def last_daily_date(ticker: str):
|
| 170 |
df = load_level(ticker, "d")
|
| 171 |
+
return None if df.empty else pd.Timestamp(df["date"].iloc[-1])
|