Spaces:
Running
Running
File size: 14,595 Bytes
d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d d88b104 d07aa1d d88b104 d07aa1d d88b104 d07aa1d 847c80d d88b104 847c80d 964ddf9 847c80d d88b104 847c80d d88b104 847c80d d88b104 847c80d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 | """
CopperMind Heatmap Backend — Project-Universe Aligned.
Universe source: config/seeds/broad_universe.csv (ticker, category, source_tag).
Hierarchy: group -> subgroup -> symbol (taxonomy derived from CSV, not Yahoo sector).
Weight fallback: marketCap -> dollarVolume (avgVolume*price) -> equalWeight(1.0).
Excluded: broad market indices (index_equity, index_global, index_vol) which are
screener correlation proxies, not displayable heatmap assets.
"""
import logging
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
import pandas as pd
from app.models import HeatmapCache
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Project taxonomy: CSV category -> (display group, display subgroup)
# These are the ONLY groups that appear in the UI.
# ---------------------------------------------------------------------------
HEATMAP_GROUP_MAP: dict[str, tuple[str, str]] = {
# Miners
"miner_major": ("Copper Miners", "Major Producers"),
"miner_mid": ("Copper Miners", "Mid-Cap"),
"miner_junior": ("Copper Miners", "Junior / Exploration"),
"miner_diversified": ("Copper Miners", "Diversified"),
# Copper-focused ETFs
"etf_copper": ("Copper & Metals ETFs", "Copper ETFs"),
"etf_metals": ("Copper & Metals ETFs", "Metals ETFs"),
"etf_miners": ("Copper & Metals ETFs", "Miner ETFs"),
# Commodity ETFs
"etf_commodity": ("Commodity ETFs", "Broad Commodity"),
"etf_sector": ("Commodity ETFs", "Sector ETFs"),
# Precious Metals
"etf_gold": ("Precious Metals", "Gold ETFs"),
"gold_etf": ("Precious Metals", "Gold ETFs"),
"etf_silver": ("Precious Metals", "Silver"),
"etf_platinum": ("Precious Metals", "Platinum & Palladium"),
"etf_palladium": ("Precious Metals", "Platinum & Palladium"),
"commodity_precious": ("Precious Metals", "Precious Futures"),
# Battery Metals
"lithium": ("Battery Metals", "Lithium"),
"rare_earth": ("Battery Metals", "Rare Earth"),
"uranium": ("Battery Metals", "Uranium"),
"ev_battery": ("Battery Metals", "EV Battery"),
# EV & Auto Demand
"auto_ev": ("EV & Auto Demand", "Electric Vehicles"),
"auto_traditional": ("EV & Auto Demand", "Traditional Auto"),
"ev_charging": ("EV & Auto Demand", "EV Charging"),
# Industrial Demand
"industrial_equipment": ("Industrial Demand", "Equipment"),
"industrial_conglom": ("Industrial Demand", "Conglomerates"),
"industrial_electrical": ("Industrial Demand", "Electrical"),
"industrial_construction": ("Industrial Demand", "Construction"),
"reit_industrial": ("Industrial Demand", "Industrial REITs"),
"infra": ("Industrial Demand", "Infrastructure"),
# Base & Materials
"materials_steel": ("Base & Materials", "Steel"),
"materials_aluminum": ("Base & Materials", "Aluminum"),
"materials_specialty": ("Base & Materials", "Specialty Materials"),
"materials_chemical": ("Base & Materials", "Chemicals"),
"commodity_base": ("Base & Materials", "Base Metal Futures"),
# Energy
"commodity_energy": ("Energy", "Energy Futures"),
"energy_major": ("Energy", "Energy Majors"),
"energy_services": ("Energy", "Energy Services"),
# Tech & Semis Demand
"tech_semi": ("Tech & Semis", "Semiconductors"),
"tech_semi_equip": ("Tech & Semis", "Semi Equipment"),
# Homebuilders
"homebuilder_etf": ("Homebuilders", "Homebuilder ETFs"),
"homebuilder": ("Homebuilders", "Homebuilders"),
# Agricultural (Demand Proxy)
"commodity_agri": ("Agricultural", "Agri Futures"),
# Macro & Rates
"macro_currency": ("Macro & Rates", "Currency"),
"rates_proxy": ("Macro & Rates", "US Rates"),
"rates_inflation": ("Macro & Rates", "TIPS / Inflation"),
# EM & China
"macro_china": ("EM & China", "China"),
"adr_china": ("EM & China", "China ADRs"),
"macro_em": ("EM & China", "EM Macro"),
"em_broad": ("EM & China", "Broad EM"),
"em_brazil": ("EM & China", "Commodity EM"),
"em_mexico": ("EM & China", "Commodity EM"),
"em_canada": ("EM & China", "Commodity EM"),
"em_southafrica": ("EM & China", "Commodity EM"),
"em_australia": ("EM & China", "Commodity EM"),
# Europe
"europe_broad": ("Europe", "Broad Europe"),
"europe_eurozone": ("Europe", "Eurozone"),
"europe_germany": ("Europe", "Germany"),
"europe_uk": ("Europe", "UK"),
# Transport (Demand Proxy)
"transport_trucking": ("Transport", "Trucking"),
"transport_rail": ("Transport", "Rail"),
"transport_cargo": ("Transport", "Cargo"),
"transport_shipping": ("Transport", "Shipping"),
# Credit & Financial (Risk Proxy)
"financial_etf": ("Credit & Financial", "Financial ETFs"),
"financial_bank": ("Credit & Financial", "Banks"),
"credit_hy": ("Credit & Financial", "High Yield"),
"credit_loans": ("Credit & Financial", "Loans"),
# Utilities
"utility_etf": ("Utilities", "Utility ETFs"),
"utility_power": ("Utilities", "Power Utilities"),
# Alternative / Crypto Risk Proxy
"crypto_proxy": ("Alternative", "Crypto Proxy"),
}
# Categories that are broad market indices — kept for screener correlation,
# excluded from the heatmap display universe entirely.
EXCLUDED_CATEGORIES: frozenset[str] = frozenset({
"index_equity", # ^GSPC, ^DJI, ^IXIC, ^RUT
"index_global", # ^STOXX50E, ^FTSE, ^N225, ^HSI
"index_vol", # ^VIX
})
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _load_universe() -> list[dict]:
"""
Load broad_universe.csv, skip comment lines, and return a list of
{ticker, category, source_tag} dicts after filtering excluded categories.
"""
for candidate in [
Path("config/seeds/broad_universe.csv"),
Path(__file__).parent.parent / "config" / "seeds" / "broad_universe.csv",
]:
if candidate.exists():
df = pd.read_csv(candidate, comment="#")
# Only keep rows that have a valid ticker (not blank / comment artifacts)
df = df.dropna(subset=["ticker"])
df["ticker"] = df["ticker"].str.strip()
df = df[df["ticker"] != ""]
# Exclude broad market index categories
df = df[~df["category"].isin(EXCLUDED_CATEGORIES)]
return df.to_dict("records")
# Absolute fallback if CSV not found
logger.warning("broad_universe.csv not found — using minimal copper fallback")
return [
{"ticker": "FCX", "category": "miner_major", "source_tag": "copper_core"},
{"ticker": "SCCO", "category": "miner_major", "source_tag": "copper_core"},
{"ticker": "BHP", "category": "miner_major", "source_tag": "copper_core"},
{"ticker": "COPX", "category": "etf_copper", "source_tag": "etf_core"},
{"ticker": "HG=F", "category": "commodity_base","source_tag": "commodity_base"},
]
def _derive_weight(info: dict, price: float) -> tuple[float, str]:
"""
Derive a sizing weight for the treemap cell.
Priority: marketCap -> dollarVolume (avgVolume * price) -> equal weight 1.0.
Returns (weight_value, weight_label).
"""
mc = info.get("marketCap")
if mc and mc > 0:
return float(mc), "Market Cap"
avg_vol = info.get("averageVolume") or info.get("regularMarketVolume") or 0
if avg_vol and avg_vol > 0 and price > 0:
dollar_vol = float(avg_vol) * price
return dollar_vol, "Dollar Volume"
return 1.0, "Equal Weight"
def _build_hierarchy(symbols: list[dict]) -> dict:
"""
Build D3-compatible squarified treemap hierarchy from project taxonomy.
Structure: root -> group -> subgroup -> leaf (symbol).
"""
groups: dict[str, dict[str, list]] = {}
for item in symbols:
grp = item.get("group", "Other")
sub = item.get("subgroup", "Other")
groups.setdefault(grp, {}).setdefault(sub, []).append(item)
root: dict = {"name": "CopperMind Universe", "children": []}
for grp_name, subs in groups.items():
grp_node: dict = {"name": grp_name, "children": []}
for sub_name, leaves in subs.items():
sub_node: dict = {"name": sub_name, "children": leaves}
grp_node["children"].append(sub_node)
root["children"].append(grp_node)
return root
def refresh_market_heatmap() -> None:
"""
Background task: fetch live quotes for the project universe and rebuild
the heatmap cache. Uses project taxonomy (not Yahoo sector/industry).
"""
from app.db import SessionLocal
with SessionLocal() as session:
cache: Optional[HeatmapCache] = session.query(HeatmapCache).first()
if not cache:
cache = HeatmapCache(
payload_json={},
cached_at=_utcnow(),
expires_at=_utcnow(),
)
session.add(cache)
cache.refresh_started_at = _utcnow()
cache.refresh_error = None
session.commit()
try:
universe_rows = _load_universe()
symbols_to_fetch = [r["ticker"] for r in universe_rows]
category_map = {r["ticker"]: r["category"] for r in universe_rows}
source_tag_map = {r["ticker"]: r.get("source_tag", "") for r in universe_rows}
logger.info("Heatmap: fetching %d project universe symbols", len(symbols_to_fetch))
import yfinance as yf
# Batch fetch with rate-limit protection
# Yahoo Finance aggressively blocks high-frequency scrapers.
# Process in batches of 50 (smaller than before) with inter-batch
# delays to stay under the radar.
import time
all_data: list[dict] = []
batch_size = 50
for i in range(0, len(symbols_to_fetch), batch_size):
batch = symbols_to_fetch[i : i + batch_size]
# Inter-batch delay to avoid Yahoo crumb invalidation
if i > 0:
time.sleep(1.5)
try:
tickers_obj = yf.Tickers(" ".join(batch))
for sym in batch:
try:
ticker = tickers_obj.tickers.get(sym)
if not ticker:
continue
info = ticker.info or {}
price = info.get("regularMarketPrice") or info.get("currentPrice")
if price is None:
# Symbol returned no live price — skip
logger.debug("Heatmap: no price for %s, skipping", sym)
continue
price = float(price)
change = float(info.get("regularMarketChangePercent") or 0.0)
short_name = info.get("shortName") or info.get("longName") or sym
weight, weight_label = _derive_weight(info, price)
cat = category_map.get(sym, "")
group, subgroup = HEATMAP_GROUP_MAP.get(cat, ("Other", cat or "Uncategorized"))
all_data.append({
"name": sym,
"shortName": short_name,
"price": round(price, 4),
"changePercent": round(change, 4),
"weight": round(weight, 2),
"weightLabel": weight_label,
"group": group,
"subgroup": subgroup,
"category": cat,
"sourceTag": source_tag_map.get(sym, ""),
})
except Exception as sym_err:
logger.debug("Heatmap: error for %s: %s", sym, sym_err)
except Exception as batch_err:
logger.warning("Heatmap: batch %d failed: %s", i // batch_size, batch_err)
logger.info("Heatmap: %d symbols with live data (of %d universe)", len(all_data), len(symbols_to_fetch))
root = _build_hierarchy(all_data)
now = _utcnow()
cache.payload_json = root
cache.cached_at = now
cache.expires_at = now + timedelta(minutes=15)
cache.refresh_started_at = None
cache.refresh_error = None
session.commit()
logger.info("Heatmap cache refreshed successfully")
except Exception as err:
logger.error("Heatmap refresh failed: %s", err, exc_info=True)
try:
c = session.query(HeatmapCache).first()
if c:
c.refresh_started_at = None
c.refresh_error = str(err)[:500]
session.commit()
except Exception:
pass
|