ranranrunforit commited on
Commit
79755bf
·
verified ·
1 Parent(s): 0c9b203

Update data_us.py

Browse files
Files changed (1) hide show
  1. data_us.py +30 -6
data_us.py CHANGED
@@ -4,7 +4,7 @@ data_us.py — US market data layer (yfinance), replacing baostock/pytdx.
4
  Levels & history limits (Yahoo Finance API constraints):
5
  daily : 10 years (weekly / monthly are resampled from daily
6
  by chan_multilevel.resample_weekly/_monthly)
7
- 60m : last 730 days
8
  30m/15m : last 60 days
9
  5m : last 60 days
10
  1m : last 7 days only → too short for Chan decomposition, NOT used.
@@ -20,6 +20,7 @@ and refreshed when stale (daily: >12h old, intraday: >2h old) or on force=True.
20
  from __future__ import annotations
21
 
22
  import os
 
23
  import time
24
  import traceback
25
 
@@ -27,12 +28,22 @@ import pandas as pd
27
 
28
  import paths
29
 
 
 
 
 
 
 
 
 
30
  CACHE_DIR = os.environ.get("CHAN_CACHE_DIR", paths.CACHE_DIR)
31
 
32
  LEVELS = {
33
- # level: (yfinance interval, period) — Yahoo's max history per interval
 
 
34
  "d": ("1d", "10y"),
35
- "60m": ("1h", "730d"),
36
  "30m": ("30m", "60d"),
37
  "15m": ("15m", "60d"),
38
  "5m": ("5m", "60d"),
@@ -91,9 +102,22 @@ def load_level(ticker: str, level: str, force: bool = False) -> pd.DataFrame:
91
  pass
92
  try:
93
  import yfinance as yf
 
94
  interval, period = LEVELS[level]
95
- raw = yf.Ticker(ticker).history(period=period, interval=interval,
96
- auto_adjust=True, actions=False)
 
 
 
 
 
 
 
 
 
 
 
 
97
  df = _normalize(raw)
98
  if len(df):
99
  df.to_parquet(path, index=False)
@@ -144,4 +168,4 @@ def prefetch(tickers, levels=FAST_LEVELS, force: bool = False, workers: int = 5,
144
 
145
  def last_daily_date(ticker: str):
146
  df = load_level(ticker, "d")
147
- return None if df.empty else pd.Timestamp(df["date"].iloc[-1])
 
4
  Levels & history limits (Yahoo Finance API constraints):
5
  daily : 10 years (weekly / monthly are resampled from daily
6
  by chan_multilevel.resample_weekly/_monthly)
7
+ 60m : last 730 days (fetched as "1h" interval with explicit start/end dates)
8
  30m/15m : last 60 days
9
  5m : last 60 days
10
  1m : last 7 days only → too short for Chan decomposition, NOT used.
 
20
  from __future__ import annotations
21
 
22
  import os
23
+ import threading
24
  import time
25
  import traceback
26
 
 
28
 
29
  import paths
30
 
31
+ # yfinance uses a shared SQLite cache (peewee) for timezone lookups.
32
+ # When multiple threads call yf.Ticker().history() simultaneously the DB
33
+ # gets locked and raises peewee.OperationalError, stalling prefetch and
34
+ # freezing the "Run analysis" button. Serialise the yfinance connect/lookup
35
+ # phase with a process-wide lock — Yahoo's own rate-limit is the real
36
+ # bottleneck anyway, so the extra serialisation costs almost nothing.
37
+ _YF_LOCK = threading.Lock()
38
+
39
  CACHE_DIR = os.environ.get("CHAN_CACHE_DIR", paths.CACHE_DIR)
40
 
41
  LEVELS = {
42
+ # level: (yfinance interval, period_or_days)
43
+ # For "60m" Yahoo requires explicit start/end dates (not a period string)
44
+ # when fetching more than ~60 days back; we pass days as an int sentinel.
45
  "d": ("1d", "10y"),
46
+ "60m": ("1h", "730d"), # use explicit start/end — "period='730d'" is rejected by Yahoo for 1h
47
  "30m": ("30m", "60d"),
48
  "15m": ("15m", "60d"),
49
  "5m": ("5m", "60d"),
 
102
  pass
103
  try:
104
  import yfinance as yf
105
+ from datetime import datetime, timedelta
106
  interval, period = LEVELS[level]
107
+ # Acquire lock before any yfinance call — the shared peewee/SQLite
108
+ # timezone cache raises "database is locked" under concurrent access.
109
+ with _YF_LOCK:
110
+ if isinstance(period, int):
111
+ # Yahoo rejects period strings for hourly data older than ~60 days.
112
+ # Use explicit start/end timestamps instead.
113
+ end_dt = datetime.utcnow()
114
+ start_dt = end_dt - timedelta(days=period)
115
+ raw = yf.Ticker(ticker).history(start=start_dt, end=end_dt,
116
+ interval=interval,
117
+ auto_adjust=True, actions=False)
118
+ else:
119
+ raw = yf.Ticker(ticker).history(period=period, interval=interval,
120
+ auto_adjust=True, actions=False)
121
  df = _normalize(raw)
122
  if len(df):
123
  df.to_parquet(path, index=False)
 
168
 
169
  def last_daily_date(ticker: str):
170
  df = load_level(ticker, "d")
171
+ return None if df.empty else pd.Timestamp(df["date"].iloc[-1])