Update data/schedule.py

data/schedule.py

@@ -1,78 +1,160 @@
 from __future__ import annotations
 
-import os
-
-APP_TITLE = "WBC Analytics Assistant"
-REFRESH_TTL_SECONDS = 30
-
-DUCKDB_PATH = "data/wbc.duckdb"
-
-DEFAULT_EDGE_THRESHOLD = 0.05
-DEFAULT_CONFIDENCE_THRESHOLD = 0.70
-
-# Official / public sources
-WBC_SCHEDULE_PAGE_URL = "https://www.mlb.com/world-baseball-classic/schedule"
-WBC_HOME_URL = "https://www.mlb.com/world-baseball-classic"
-WBC_STATS_URL = "https://www.mlb.com/world-baseball-classic/stats"
-WBC_STANDINGS_URL = "https://www.mlb.com/world-baseball-classic/standings"
-
-# Baseball Savant WBC Statcast
-WBC_STATCAST_SEARCH_URL = "https://baseballsavant.mlb.com/statcast-search-world-baseball-classic"
-STATCAST_SEARCH_URL = "https://baseballsavant.mlb.com/statcast_search/csv"
-
-# MLB enrichment / fallback
-MLB_SCHEDULE_URL = "https://statsapi.mlb.com/api/v1/schedule"
-MLB_TEAMS_URL = "https://statsapi.mlb.com/api/v1/teams"
-
-# Weather
-OPENWEATHER_URL = "https://api.openweathermap.org/data/2.5/weather"
-
-# Odds
-ODDS_SPORT_KEY = "baseball_mlb"
-ODDS_BASE_URL = "https://api.the-odds-api.com/v4"
-ODDS_REGIONS = "us"
-ODDS_FEATURED_MARKETS = "h2h,spreads,totals"
-ODDS_FORMAT = "american"
-
-ODDS_API_KEY = os.getenv("ODDS_API_KEY", "")
-OPENWEATHER_API_KEY = os.getenv("OPENWEATHER_API_KEY", "")
-
-SUPPORTED_BOOKS = [
-    "DraftKings",
-    "FanDuel",
-    "BetMGM",
-    "Caesars",
-    "bet365",
-    "Pinnacle",
-]
-
-WBC_2026_VENUES = {
-    "Hiram Bithorn Stadium": {"lat": 18.3982, "lon": -66.0600, "city": "San Juan"},
-    "Daikin Park": {"lat": 29.7573, "lon": -95.3555, "city": "Houston"},
-    "Tokyo Dome": {"lat": 35.7056, "lon": 139.7519, "city": "Tokyo"},
-    "loanDepot park": {"lat": 25.7781, "lon": -80.2197, "city": "Miami"},
+import json
+import re
+from datetime import datetime
+from typing import Any
+
+import pandas as pd
+import requests
+
+from config.settings import WBC_SCHEDULE_PAGE_URL
+
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept-Language": "en-US,en;q=0.9",
 }
 
-WBC_TEAMS = [
-    "Australia",
-    "Brazil",
-    "Canada",
-    "China",
-    "Chinese Taipei",
-    "Colombia",
-    "Cuba",
-    "Czech Republic",
-    "Dominican Republic",
-    "Great Britain",
-    "Israel",
-    "Italy",
-    "Japan",
-    "Korea",
-    "Mexico",
-    "Netherlands",
-    "Nicaragua",
-    "Panama",
-    "Puerto Rico",
-    "United States",
-    "Venezuela",
-]
+
+def _safe_text(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def _extract_next_data_json(html: str) -> dict[str, Any] | None:
+    patterns = [
+        r'<script id="__NEXT_DATA__" type="application/json">(.+?)</script>',
+        r'window\.__PRELOADED_STATE__\s*=\s*({.+?});',
+    ]
+
+    for pattern in patterns:
+        match = re.search(pattern, html, flags=re.DOTALL)
+        if match:
+            raw = match.group(1)
+            try:
+                return json.loads(raw)
+            except json.JSONDecodeError:
+                continue
+    return None
+
+
+def _rows_from_embedded_json(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+
+    def walk(obj: Any) -> None:
+        if isinstance(obj, dict):
+            keys = set(obj.keys())
+
+            possible_game = (
+                ("homeTeam" in keys or "home_team" in keys or "home" in keys)
+                and ("awayTeam" in keys or "away_team" in keys or "away" in keys)
+            )
+
+            if possible_game:
+                home_team = (
+                    obj.get("homeTeam")
+                    or obj.get("home_team")
+                    or (obj.get("home") or {}).get("name")
+                    or ""
+                )
+                away_team = (
+                    obj.get("awayTeam")
+                    or obj.get("away_team")
+                    or (obj.get("away") or {}).get("name")
+                    or ""
+                )
+                venue = obj.get("venue") or obj.get("venueName") or ""
+                status = obj.get("status") or obj.get("detailedState") or ""
+                game_id = obj.get("id") or obj.get("gamePk") or obj.get("game_id") or ""
+                game_date = obj.get("gameDate") or obj.get("date") or obj.get("commence_time") or ""
+                home_score = obj.get("homeScore")
+                away_score = obj.get("awayScore")
+
+                if home_team and away_team:
+                    rows.append(
+                        {
+                            "fetched_at": datetime.utcnow(),
+                            "game_id": _safe_text(game_id),
+                            "game_date": _safe_text(game_date),
+                            "status": _safe_text(status),
+                            "away_team": _safe_text(away_team),
+                            "home_team": _safe_text(home_team),
+                            "away_score": away_score,
+                            "home_score": home_score,
+                            "venue": _safe_text(venue),
+                        }
+                    )
+
+            for value in obj.values():
+                walk(value)
+
+        elif isinstance(obj, list):
+            for item in obj:
+                walk(item)
+
+    walk(payload)
+    return rows
+
+
+def _rows_from_schedule_text(html: str) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+
+    lines = [line.strip() for line in html.splitlines() if line.strip()]
+    joined = "\n".join(lines)
+
+    matches = re.findall(r"([A-Z][A-Za-z .]+)\s+vs\.\s+([A-Z][A-Za-z .]+)", joined)
+    seen = set()
+
+    for away_team, home_team in matches:
+        key = (away_team, home_team)
+        if key in seen:
+            continue
+        seen.add(key)
+        rows.append(
+            {
+                "fetched_at": datetime.utcnow(),
+                "game_id": "",
+                "game_date": "",
+                "status": "",
+                "away_team": away_team.strip(),
+                "home_team": home_team.strip(),
+                "away_score": None,
+                "home_score": None,
+                "venue": "",
+            }
+        )
+
+    return rows
+
+
+def fetch_wbc_schedule() -> pd.DataFrame:
+    response = requests.get(WBC_SCHEDULE_PAGE_URL, headers=HEADERS, timeout=30)
+    response.raise_for_status()
+    html = response.text
+
+    payload = _extract_next_data_json(html)
+    rows: list[dict[str, Any]] = []
+
+    if payload is not None:
+        rows = _rows_from_embedded_json(payload)
+
+    if not rows:
+        rows = _rows_from_schedule_text(html)
+
+    df = pd.DataFrame(rows)
+
+    if df.empty:
+        return df
+
+    df = df.drop_duplicates(subset=["game_id", "away_team", "home_team", "game_date"]).copy()
+
+    if "game_date" in df.columns:
+        try:
+            df["game_date_sort"] = pd.to_datetime(df["game_date"], errors="coerce")
+            df = df.sort_values(["game_date_sort", "away_team", "home_team"]).drop(columns=["game_date_sort"])
+        except Exception:
+            pass
+
+    return df.reset_index(drop=True)