""" data_fetcher.py =============== Centralized data-fetching layer for the MLB Pregame Dashboard. Sources: - pybaseball → Baseball Savant (Statcast), FanGraphs splits - requests → RotoGrinders (lineups + weather), Baseball Reference - MLB Stats API (statsapi) → game schedule, live game state All public functions return plain Python dicts / DataFrames so the UI layer has zero direct network dependencies. Performance notes: - Results are cached with a TTL so repeated renders don't re-hit APIs. - Heavy pybaseball calls (statcast) are run once per pitcher/date combo. - Caches are module-level dicts; swap for Redis/disk if you scale. """ import datetime import functools import re import time import warnings from typing import Optional import numpy as np import pandas as pd import requests # ── optional deps (graceful degradation if not installed) ───────────────────── try: import pybaseball as pyb pyb.cache.enable() # pybaseball's built-in disk cache HAS_PYBASEBALL = True except ImportError: HAS_PYBASEBALL = False warnings.warn("pybaseball not installed – Statcast data will be unavailable.") try: import statsapi # MLB-StatsAPI HAS_STATSAPI = True except ImportError: HAS_STATSAPI = False warnings.warn("MLB-StatsAPI not installed – schedule data will be unavailable.") # ── League-average constants (2024 season) ─────────────────────────────────── # Update these each season. Used for color-coding relative performance. LEAGUE_AVG = { "xwOBA": 0.318, "K_pct": 0.226, # 22.6 % "BB_pct": 0.082, # 8.2 % # Fastball velo league avg by role "SP_FB_velo": 93.9, "RP_FB_velo": 95.2, } SMALL_SAMPLE_PA = 100 # PA threshold for the ⚠ sample-size flag # ── Simple in-memory cache ──────────────────────────────────────────────────── _CACHE: dict = {} _CACHE_TTL = 300 # seconds def _cache_get(key: str): entry = _CACHE.get(key) if entry and (time.time() - entry["ts"] < _CACHE_TTL): return entry["val"] return None def _cache_set(key: str, val): _CACHE[key] = {"ts": time.time(), "val": val} def _cached(ttl_override: Optional[int] = None): """Decorator: cache function result by (fn_name, *args, **kwargs).""" def decorator(fn): @functools.wraps(fn) def wrapper(*args, **kwargs): key = f"{fn.__name__}:{args}:{sorted(kwargs.items())}" hit = _cache_get(key) if hit is not None: return hit result = fn(*args, **kwargs) _CACHE[key] = {"ts": time.time(), "val": result} return result return wrapper return decorator # ══════════════════════════════════════════════════════════════════════════════ # SCHEDULE / GAME STATE # ══════════════════════════════════════════════════════════════════════════════ @_cached() def get_schedule(date: datetime.date) -> list[dict]: """ Return a list of game dicts for the given date via MLB StatsAPI. Each dict contains: game_pk, away_team, home_team, away_abbr, home_abbr, status ('Future'|'Today'|'Live'|'Past'), game_time (local), away_starter_id, home_starter_id, away_starter_name, home_starter_name, inning, outs, score_away, score_home, venue """ if not HAS_STATSAPI: return _mock_schedule(date) today = datetime.date.today() date_str = date.strftime("%m/%d/%Y") try: raw = statsapi.schedule(date=date_str, sportId=1) except Exception as e: warnings.warn(f"StatsAPI schedule error: {e}") return [] games = [] for g in raw: # Determine status label if date < today: status = "Past" elif date > today: status = "Future" elif g.get("status") in ("In Progress", "Warmup", "Pre-Game"): status = "Live" else: status = "Today" games.append({ "game_pk": g.get("game_id"), "away_team": g.get("away_name", ""), "home_team": g.get("home_name", ""), "away_abbr": g.get("away_id", ""), # numeric; resolve below "home_abbr": _team_abbr(g.get("away_name", "")), # name→abbr "home_abbr": _team_abbr(g.get("home_name", "")), "away_abbr": _team_abbr(g.get("away_name", "")), "status": status, "game_time": g.get("game_datetime", ""), "away_starter_name": g.get("away_probable_pitcher", "TBD"), "home_starter_name": g.get("home_probable_pitcher", "TBD"), "away_starter_id": g.get("away_probable_pitcher_id"), "home_starter_id": g.get("home_probable_pitcher_id"), "inning": g.get("current_inning"), "outs": g.get("outs"), "score_away": g.get("away_score"), "score_home": g.get("home_score"), "venue": g.get("venue_name", ""), }) return games def _mock_schedule(date: datetime.date) -> list[dict]: """Fallback schedule when statsapi is unavailable — returns two placeholder games.""" today = datetime.date.today() status = "Today" if date == today else ("Future" if date > today else "Past") return [ { "game_pk": 1, "away_team": "Los Angeles Dodgers", "home_team": "Philadelphia Phillies", "away_abbr": "LAD", "home_abbr": "PHI", "status": status, "game_time": "7:08 PM ET", "away_starter_name": "Y. Yamamoto", "home_starter_name": "C. Sanchez", "away_starter_id": None, "home_starter_id": None, "inning": None, "outs": None, "score_away": None, "score_home": None, "venue": "Citizens Bank Park", }, { "game_pk": 2, "away_team": "New York Yankees", "home_team": "Boston Red Sox", "away_abbr": "NYY", "home_abbr": "BOS", "status": status, "game_time": "7:10 PM ET", "away_starter_name": "G. Cole", "home_starter_name": "B. Crawford", "away_starter_id": None, "home_starter_id": None, "inning": None, "outs": None, "score_away": None, "score_home": None, "venue": "Fenway Park", }, ] # ══════════════════════════════════════════════════════════════════════════════ # WEATHER (RotoGrinders — today only) # ══════════════════════════════════════════════════════════════════════════════ @_cached(ttl_override=600) def get_weather(venue: str) -> dict: """ Scrape weather data from RotoGrinders for today's games. Returns dict: {temp_f, wind_mph, wind_dir, conditions, precip_pct} Falls back to empty dict if unavailable / not today. """ try: url = "https://rotogrinders.com/weather/mlb" resp = requests.get(url, timeout=10, headers={"User-Agent": "Mozilla/5.0"}) resp.raise_for_status() # RotoGrinders renders weather in a JS-heavy page; parse what we can. # Look for venue name in the raw HTML and grab surrounding numbers. html = resp.text # Simplified regex extraction — adjust xpath/pattern to match live HTML pattern = re.compile( rf"{re.escape(venue[:12])}.{{0,300}}?(\d{{2,3}})°.{{0,100}}?(\d+)\s*mph", re.DOTALL | re.IGNORECASE, ) m = pattern.search(html) if m: return {"temp_f": int(m.group(1)), "wind_mph": int(m.group(2)), "wind_dir": "", "conditions": "See RotoGrinders", "precip_pct": None} except Exception: pass return {} # ══════════════════════════════════════════════════════════════════════════════ # LINEUPS # ══════════════════════════════════════════════════════════════════════════════ @_cached() def get_lineups(game_pk: int, away_abbr: str, home_abbr: str, date: datetime.date) -> dict: """ Attempt to fetch confirmed lineups via MLB StatsAPI. Falls back to projected lineups from RotoGrinders. Falls back to typical order from Baseball Reference. Returns: { "away": { "official": bool, "players": [player_dict, ...] }, "home": { "official": bool, "players": [player_dict, ...] }, "away_bench": [player_dict, ...], "home_bench": [player_dict, ...], } where player_dict: {order, name, player_id, bats, pos} """ # ── Try MLB StatsAPI live lineup ────────────────────────────────────────── if HAS_STATSAPI and game_pk: try: boxscore = statsapi.boxscore_data(game_pk) away_lineup = _parse_statsapi_lineup(boxscore, "away") home_lineup = _parse_statsapi_lineup(boxscore, "home") if away_lineup and home_lineup: return { "away": {"official": True, "players": away_lineup["starters"]}, "home": {"official": True, "players": home_lineup["starters"]}, "away_bench": away_lineup["bench"], "home_bench": home_lineup["bench"], } except Exception: pass # ── Fall back to RotoGrinders projected lineups ─────────────────────────── rg = _fetch_rotogrinders_lineups(away_abbr, home_abbr) if rg: return rg # ── Final fallback: return empty structure with flag ────────────────────── return { "away": {"official": False, "players": []}, "home": {"official": False, "players": []}, "away_bench": [], "home_bench": [], } def _parse_statsapi_lineup(boxscore: dict, side: str) -> Optional[dict]: """Extract starters and bench from StatsAPI boxscore_data.""" try: team = boxscore[side] players = team.get("players", {}) starters, bench = [], [] for pid, p in players.items(): info = { "order": p.get("battingOrder", 0), "name": p["person"]["fullName"], "player_id": p["person"]["id"], "bats": p.get("batSide", {}).get("code", "?"), "pos": p.get("position", {}).get("abbreviation", ""), } if p.get("battingOrder") and int(p["battingOrder"]) % 100 == 0: starters.append(info) else: bench.append(info) starters.sort(key=lambda x: x["order"]) return {"starters": starters, "bench": bench} except Exception: return None def _fetch_rotogrinders_lineups(away: str, home: str) -> Optional[dict]: """ Scrape projected lineups from RotoGrinders. Returns None on failure. """ try: url = "https://rotogrinders.com/lineups/mlb" resp = requests.get(url, timeout=10, headers={"User-Agent": "Mozilla/5.0"}) resp.raise_for_status() # TODO: parse the JS-rendered content. # RotoGrinders uses React; consider using their undocumented JSON endpoint: # https://rotogrinders.com/api/lineups?sport=mlb&date=YYYY-MM-DD # For now return None to trigger the next fallback. return None except Exception: return None # ══════════════════════════════════════════════════════════════════════════════ # HITTER SPLITS + RECENT PERFORMANCE # ══════════════════════════════════════════════════════════════════════════════ @_cached() def get_hitter_splits(player_id: int, season: int = None) -> dict: """ Fetch batter L/R splits (xwOBA, PA) from Baseball Savant via pybaseball. Returns: { "vs_L": {"xwOBA": float, "PA": int}, "vs_R": {"xwOBA": float, "PA": int} } """ if season is None: season = datetime.date.today().year if not HAS_PYBASEBALL: return _mock_hitter_splits() try: # statcast_batter returns a raw Statcast DataFrame df = pyb.statcast_batter( start_dt=f"{season}-03-01", end_dt=f"{season}-10-01", player_id=player_id, ) if df is None or df.empty: return _mock_hitter_splits() result = {} for hand in ("L", "R"): subset = df[df["p_throws"] == hand] result[f"vs_{hand}"] = { "xwOBA": _safe_mean(subset, "estimated_woba_using_speedangle"), "PA": len(subset), } return result except Exception as e: warnings.warn(f"Hitter splits error for {player_id}: {e}") return _mock_hitter_splits() def _mock_hitter_splits() -> dict: return { "vs_L": {"xwOBA": round(np.random.uniform(0.28, 0.40), 3), "PA": np.random.randint(50, 300)}, "vs_R": {"xwOBA": round(np.random.uniform(0.28, 0.40), 3), "PA": np.random.randint(80, 400)}, } @_cached() def get_hitter_recent(player_id: int, days: int = 7) -> dict: """ Fetch last N days of batter performance via Statcast. Returns: { "H": int, "AB": int, "HR": int, "K": int, "xwOBA": float } """ if not HAS_PYBASEBALL: return _mock_hitter_recent() end = datetime.date.today() start = end - datetime.timedelta(days=days) try: df = pyb.statcast_batter( start_dt=start.strftime("%Y-%m-%d"), end_dt=end.strftime("%Y-%m-%d"), player_id=player_id, ) if df is None or df.empty: return _mock_hitter_recent() hits = df[df["events"].isin(["single","double","triple","home_run"])] ab_events = ["single","double","triple","home_run","strikeout", "field_out","grounded_into_double_play","force_out", "fielders_choice_out","sac_fly","double_play"] ab = df[df["events"].isin(ab_events)] return { "H": len(hits), "AB": len(ab), "HR": int(df["events"].eq("home_run").sum()), "K": int(df["events"].eq("strikeout").sum()), "xwOBA": _safe_mean(df, "estimated_woba_using_speedangle"), } except Exception as e: warnings.warn(f"Hitter recent error for {player_id}: {e}") return _mock_hitter_recent() def _mock_hitter_recent() -> dict: ab = np.random.randint(18, 30) return { "H": np.random.randint(3, 10), "AB": ab, "HR": np.random.randint(0, 3), "K": np.random.randint(2, 9), "xwOBA": round(np.random.uniform(0.25, 0.45), 3), } # ══════════════════════════════════════════════════════════════════════════════ # PITCHER DATA # ══════════════════════════════════════════════════════════════════════════════ @_cached() def get_pitcher_game_log(player_id: int, n_starts: int = 30) -> pd.DataFrame: """ Fetch pitcher's recent game log (last N starts, max 12 months). Returns DataFrame with columns: date, IP (innings_pitched float), pitches, GS (bool) """ if not HAS_PYBASEBALL: return _mock_pitcher_game_log() end = datetime.date.today() start = end - datetime.timedelta(days=365) try: df = pyb.pitching_stats_range( start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d"), ) # Filter to this pitcher df = df[df["IDfg"] == player_id].copy() df["date"] = pd.to_datetime(df["Date"]) df = df[df["GS"] == 1].sort_values("date").tail(n_starts) df["IP_float"] = df["IP"].apply(_ip_to_float) return df[["date", "IP_float", "Pitches"]].rename( columns={"IP_float": "IP", "Pitches": "pitches"} ) except Exception as e: warnings.warn(f"Pitcher game log error: {e}") return _mock_pitcher_game_log() def _mock_pitcher_game_log(n: int = 22) -> pd.DataFrame: dates = pd.date_range(end=datetime.date.today(), periods=n, freq="5D") ip = np.random.choice([5.0, 5.1, 5.2, 6.0, 6.1, 6.2, 7.0, 7.1], size=n, p=[0.05, 0.07, 0.08, 0.2, 0.15, 0.15, 0.2, 0.1]) pitches = (ip * 16 + np.random.randint(-8, 8, n)).astype(int) return pd.DataFrame({"date": dates, "IP": ip, "pitches": pitches}) @_cached() def get_pitcher_splits(player_id: int, season: int = None) -> dict: """ Fetch pitcher L/R splits: K%, BB%, xwOBA via pybaseball Statcast. Returns: { "vs_L": {"K_pct": float, "BB_pct": float, "xwOBA": float, "PA": int}, "vs_R": { ... } } """ if season is None: season = datetime.date.today().year if not HAS_PYBASEBALL: return _mock_pitcher_splits() try: df = pyb.statcast_pitcher( start_dt=f"{season}-03-01", end_dt=f"{season}-10-01", player_id=player_id, ) if df is None or df.empty: return _mock_pitcher_splits() result = {} for hand in ("L", "R"): sub = df[df["stand"] == hand] pa = len(sub[sub["events"].notna()]) k = int(sub["events"].eq("strikeout").sum()) bb = int(sub["events"].eq("walk").sum()) result[f"vs_{hand}"] = { "K_pct": k / pa if pa else 0, "BB_pct": bb / pa if pa else 0, "xwOBA": _safe_mean(sub, "estimated_woba_using_speedangle"), "PA": pa, } return result except Exception as e: warnings.warn(f"Pitcher splits error: {e}") return _mock_pitcher_splits() def _mock_pitcher_splits() -> dict: return { "vs_L": {"K_pct": round(np.random.uniform(0.18, 0.32), 3), "BB_pct": round(np.random.uniform(0.06, 0.12), 3), "xwOBA": round(np.random.uniform(0.28, 0.38), 3), "PA": np.random.randint(80, 200)}, "vs_R": {"K_pct": round(np.random.uniform(0.18, 0.32), 3), "BB_pct": round(np.random.uniform(0.06, 0.12), 3), "xwOBA": round(np.random.uniform(0.28, 0.38), 3), "PA": np.random.randint(80, 200)}, } @_cached() def get_pitcher_velo_by_inning(player_id: int, season: int = None) -> dict: """ Fetch pitcher fastball velocity by inning (innings 1-7). Returns dict: { inning: [list of velo floats] } for box plots. Only includes four-seam (FF) and sinker (SI) pitches. """ if season is None: season = datetime.date.today().year if not HAS_PYBASEBALL: return _mock_velo_by_inning() try: df = pyb.statcast_pitcher( start_dt=f"{season}-03-01", end_dt=f"{season}-10-01", player_id=player_id, ) if df is None or df.empty: return _mock_velo_by_inning() fb = df[df["pitch_type"].isin(["FF", "SI"]) & df["release_speed"].notna()] result = {} for inning in range(1, 8): velos = fb[fb["inning"] == inning]["release_speed"].tolist() if velos: result[inning] = velos return result except Exception as e: warnings.warn(f"Pitcher velo error: {e}") return _mock_velo_by_inning() def _mock_velo_by_inning() -> dict: base = np.random.uniform(92, 96) result = {} for i in range(1, 8): n = np.random.randint(15, 40) # Slight velocity drop as game progresses mean = base - (i - 1) * 0.18 result[i] = list(np.clip(np.random.normal(mean, 0.8, n), mean - 3, mean + 3).round(1)) return result # ══════════════════════════════════════════════════════════════════════════════ # BULLPEN # ══════════════════════════════════════════════════════════════════════════════ @_cached() def get_bullpen(team_abbr: str, season: int = None) -> list[dict]: """ Return bullpen arms for a team. Each dict: name, player_id, throws, splits (same format as pitcher_splits), typical_entry_inning (dict {6: pct, 7: pct, 8: pct, 9+: pct}), pitches_last_3_days: [day_1_pitches, day_2_pitches, day_3_pitches] """ if not HAS_PYBASEBALL: return _mock_bullpen(team_abbr) try: year = season or datetime.date.today().year stats = pyb.pitching_stats(year, qual=0) # Filter to team relievers (GS == 0 or very low) team_stats = stats[ (stats["Team"] == team_abbr) & (stats["GS"] / stats["G"].replace(0, 1) < 0.3) ].head(8) arms = [] for _, row in team_stats.iterrows(): fgid = row.get("IDfg") arms.append({ "name": row.get("Name", "Unknown"), "player_id": fgid, "throws": row.get("Throws", "R"), "splits": get_pitcher_splits(fgid) if fgid else _mock_pitcher_splits(), "typical_entry_inning": _get_entry_inning_dist(fgid, year), "pitches_last_3_days": _get_recent_pitches(fgid, days=3), }) return arms except Exception as e: warnings.warn(f"Bullpen error for {team_abbr}: {e}") return _mock_bullpen(team_abbr) def _get_entry_inning_dist(player_id: int, season: int) -> dict: """ Return distribution of entry innings (6,7,8,9+) from Statcast appearance data. """ if not HAS_PYBASEBALL: return _mock_entry_inning() try: df = pyb.statcast_pitcher( start_dt=f"{season}-03-01", end_dt=f"{season}-10-01", player_id=player_id, ) if df is None or df.empty: return _mock_entry_inning() # First pitch of each game appearance = entry inning appearances = df.groupby("game_date")["inning"].min() counts = {6: 0, 7: 0, 8: 0, "9+": 0} for inn in appearances: if inn == 6: counts[6] += 1 elif inn == 7: counts[7] += 1 elif inn == 8: counts[8] += 1 else: counts["9+"] += 1 total = sum(counts.values()) or 1 return {k: v / total for k, v in counts.items()} except Exception: return _mock_entry_inning() def _mock_entry_inning() -> dict: vals = np.random.dirichlet([1, 2, 3, 1]) return {6: round(vals[0], 2), 7: round(vals[1], 2), 8: round(vals[2], 2), "9+": round(vals[3], 2)} def _get_recent_pitches(player_id: int, days: int = 3) -> list[int]: """Return pitch counts for each of the last `days` days (0 if didn't pitch).""" if not HAS_PYBASEBALL: return [np.random.choice([0, 0, 0, 12, 18, 22, 28]) for _ in range(days)] try: end = datetime.date.today() start = end - datetime.timedelta(days=days - 1) df = pyb.statcast_pitcher( start_dt=start.strftime("%Y-%m-%d"), end_dt=end.strftime("%Y-%m-%d"), player_id=player_id, ) result = [] for i in range(days - 1, -1, -1): # today, yesterday, 2 days ago day = end - datetime.timedelta(days=i) day_df = df[df["game_date"] == day.strftime("%Y-%m-%d")] result.append(len(day_df)) return result except Exception: return [0] * days def _mock_bullpen(team_abbr: str) -> list[dict]: """Generate mock bullpen data for development.""" names = [ ("Closer A", "R"), ("Setup B", "L"), ("Setup C", "R"), ("Middle D", "R"), ("Middle E", "L"), ("Swing F", "R"), ] arms = [] for name, throws in names: arms.append({ "name": f"{team_abbr} {name}", "player_id": None, "throws": throws, "splits": _mock_pitcher_splits(), "typical_entry_inning": _mock_entry_inning(), "pitches_last_3_days": [np.random.choice([0, 0, 0, 15, 22, 28]) for _ in range(3)], }) return arms # ══════════════════════════════════════════════════════════════════════════════ # TEAM COLORS + ABBREVIATION HELPERS # ══════════════════════════════════════════════════════════════════════════════ # Primary and secondary hex colors for all 30 MLB teams TEAM_COLORS: dict[str, tuple[str, str]] = { "ARI": ("#A71930", "#E3D4AD"), "ATL": ("#CE1141", "#13274F"), "BAL": ("#DF4601", "#000000"), "BOS": ("#BD3039", "#0C2340"), "CHC": ("#0E3386", "#CC3433"), "CWS": ("#27251F", "#C4CED4"), "CIN": ("#C6011F", "#000000"), "CLE": ("#00385D", "#E50022"), "COL": ("#33006F", "#C4CED4"), "DET": ("#0C2340", "#FA4616"), "HOU": ("#002D62", "#EB6E1F"), "KC": ("#004687", "#C09A5B"), "LAA": ("#BA0021", "#003263"), "LAD": ("#005A9C", "#EF3E42"), "MIA": ("#00A3E0", "#EF3340"), "MIL": ("#12284B", "#FFC52F"), "MIN": ("#002B5C", "#D31145"), "NYM": ("#002D72", "#FF5910"), "NYY": ("#003087", "#C4CED4"), "OAK": ("#003831", "#EFB21E"), "PHI": ("#E81828", "#002D72"), "PIT": ("#27251F", "#FDB827"), "SD": ("#2F241D", "#FFC425"), "SEA": ("#0C2C56", "#005C5C"), "SF": ("#FD5A1E", "#27251F"), "STL": ("#C41E3A", "#0C2340"), "TB": ("#092C5C", "#8FBCE6"), "TEX": ("#003278", "#C0111F"), "TOR": ("#134A8E", "#E8291C"), "WSH": ("#AB0003", "#14225A"), } _NAME_TO_ABBR = { "Arizona Diamondbacks": "ARI", "Atlanta Braves": "ATL", "Baltimore Orioles": "BAL", "Boston Red Sox": "BOS", "Chicago Cubs": "CHC", "Chicago White Sox": "CWS", "Cincinnati Reds": "CIN", "Cleveland Guardians": "CLE", "Colorado Rockies": "COL", "Detroit Tigers": "DET", "Houston Astros": "HOU", "Kansas City Royals": "KC", "Los Angeles Angels": "LAA", "Los Angeles Dodgers": "LAD", "Miami Marlins": "MIA", "Milwaukee Brewers": "MIL", "Minnesota Twins": "MIN", "New York Mets": "NYM", "New York Yankees": "NYY", "Oakland Athletics": "OAK", "Philadelphia Phillies": "PHI", "Pittsburgh Pirates": "PIT", "San Diego Padres": "SD", "Seattle Mariners": "SEA", "San Francisco Giants": "SF", "St. Louis Cardinals": "STL", "Tampa Bay Rays": "TB", "Texas Rangers": "TEX", "Toronto Blue Jays": "TOR", "Washington Nationals": "WSH", } def _team_abbr(name: str) -> str: return _NAME_TO_ABBR.get(name, name[:3].upper()) # ══════════════════════════════════════════════════════════════════════════════ # UTILITY # ══════════════════════════════════════════════════════════════════════════════ def _safe_mean(df: pd.DataFrame, col: str) -> float: """Return mean of a column, handling NaN/empty gracefully.""" try: vals = df[col].dropna() return round(float(vals.mean()), 3) if len(vals) else 0.0 except Exception: return 0.0 def _ip_to_float(ip) -> float: """Convert '6.2' innings-pitched notation to decimal (6.2 = 6 + 2/3).""" try: s = str(ip) whole, frac = s.split(".") if "." in s else (s, "0") return int(whole) + int(frac) / 3 except Exception: return 0.0