Add multi-book scrape fallback for HR props + restore Odds API market key

The Odds API indexes h2h first, player props later (24-48h before game
time). FanDuel/DK have props posted but Odds API hasn't crawled them yet.

Changes:
- provider_theoddsapi.py: restore markets=batter_home_runs (correct key),
remove DIAGNOSTIC block (served its purpose)
- provider_scrape.py (new): ScrapeFallbackProvider with independent
fetchers for DraftKings, FanDuel, BetMGM, and Caesars using their
semi-public JSON APIs. Each book's failure is caught independently.
DraftKings uses dynamic category discovery (survives ID changes).
- live_prop_odds.py: change from concat-all to stop-at-first-success
chaining. Register ScrapeFallbackProvider after TheOddsAPIProvider.
Scraper only runs when Odds API returns empty.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

data/live_prop_odds.py

@@ -4,6 +4,7 @@ import pandas as pd
 
 from config.settings import ENABLE_ENTERPRISE_PROVIDER
 from data.provider_enterprise import EnterpriseMarketProvider
+from data.provider_scrape import ScrapeFallbackProvider
 from data.provider_theoddsapi import TheOddsAPIProvider
 from utils.logger import logger
 
@@ -76,8 +77,8 @@ def fetch_all_upcoming_hr_props(
         providers.append(EnterpriseMarketProvider())
 
     providers.append(TheOddsAPIProvider())
+    providers.append(ScrapeFallbackProvider())   # fallback if Odds API returns empty
 
-    frames = []
     for provider in providers:
         try:
             fetch_fn = getattr(provider, "fetch_all_upcoming_hr_props", None)
@@ -85,16 +86,12 @@ def fetch_all_upcoming_hr_props(
                 continue
             df = fetch_fn(sportsbooks=sportsbooks)
             if not df.empty:
-                frames.append(df)
+                return normalize_prop_odds(df)   # stop at first provider that returns data
         except Exception as e:
             logger.warning(f"[odds_provider_fetch] failure: {e}", exc_info=True)
             continue
 
-    if not frames:
-        return pd.DataFrame()
-
-    merged = pd.concat(frames, ignore_index=True)
-    return normalize_prop_odds(merged)
+    return pd.DataFrame()
 
 
 def fetch_live_prop_odds(

data/provider_scrape.py

@@ -0,0 +1,334 @@
+"""
+data/provider_scrape.py
+
+Fallback scraper for HR props when The Odds API hasn't yet indexed player props.
+Hits each book's semi-public JSON API directly using requests only (no browser).
+
+Books: DraftKings, FanDuel, BetMGM, Caesars
+Each book's fetch is independent — one failure does not block the others.
+Results are concatenated across all books that respond successfully.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import pandas as pd
+import requests
+
+from data.market_provider_base import MarketProviderBase
+from data.odds_name_map import map_odds_name_to_model_name
+
+_log = logging.getLogger(__name__)
+
+_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept": "application/json, text/plain, */*",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+_DK_BASE = "https://sportsbook-nash.draftkings.com/api/odds/v1/league/84240"
+_FD_BASE = "https://sbapi.il.sportsbook.fanduel.com/api"
+_FD_AK = "FhMFpcPWXMeyZxOx"
+_BETMGM_ACCESS_ID = "NmFjNjYzMmQtMmZlNS00MDQ3LWIzZjctNGMxMjhmNjNmNWVm"
+
+
+def _make_row(
+    provider_name: str,
+    event_id: str,
+    commence_time: str,
+    away_team: str,
+    home_team: str,
+    sportsbook: str,
+    sportsbook_key: str,
+    player_name_raw: str,
+    odds_american: int,
+) -> dict[str, Any]:
+    return {
+        "provider": provider_name,
+        "event_id": event_id,
+        "commence_time": commence_time,
+        "away_team": away_team,
+        "home_team": home_team,
+        "sportsbook": sportsbook,
+        "sportsbook_key": sportsbook_key,
+        "market_key": "batter_home_runs",
+        "market": "hr",
+        "player_name_raw": player_name_raw,
+        "player_name": map_odds_name_to_model_name(player_name_raw),
+        "odds_american": odds_american,
+        "line": 0.5,
+    }
+
+
+class ScrapeFallbackProvider(MarketProviderBase):
+    provider_name = "scrape_fallback"
+
+    def fetch_live_prop_odds(
+        self,
+        game_context,
+        sportsbooks=None,
+        markets=None,
+    ) -> pd.DataFrame:
+        return pd.DataFrame()
+
+    def fetch_all_upcoming_hr_props(self, sportsbooks=None) -> pd.DataFrame:
+        frames = []
+        for fetch_fn in [
+            self._fetch_draftkings,
+            self._fetch_fanduel,
+            self._fetch_betmgm,
+            self._fetch_caesars,
+        ]:
+            try:
+                df = fetch_fn()
+                if not df.empty:
+                    frames.append(df)
+                    _log.warning(
+                        "[scrape_fallback] %s returned %d rows",
+                        fetch_fn.__name__,
+                        len(df),
+                    )
+                else:
+                    _log.warning("[scrape_fallback] %s returned 0 rows", fetch_fn.__name__)
+            except Exception as exc:
+                _log.warning("[scrape_fallback] %s failed: %s", fetch_fn.__name__, exc)
+
+        result = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
+        _log.warning("[scrape_fallback] SUMMARY total_rows=%d", len(result))
+        return result
+
+    # ---------------------------------------------------------------------------
+    # DraftKings
+    # ---------------------------------------------------------------------------
+
+    def _fetch_draftkings(self) -> pd.DataFrame:
+        cat_id, sub_id = self._discover_dk_hr_category()
+        if not cat_id or not sub_id:
+            _log.warning("[dk_scrape] HR subcategory not found in categories response")
+            return pd.DataFrame()
+        url = f"{_DK_BASE}/categories/{cat_id}/subcategories/{sub_id}"
+        r = requests.get(url, headers=_HEADERS, timeout=20)
+        _log.warning("[dk_scrape] HTTP %s cat=%s sub=%s", r.status_code, cat_id, sub_id)
+        r.raise_for_status()
+        return self._parse_dk(r.json())
+
+    def _discover_dk_hr_category(self) -> tuple[int | None, int | None]:
+        r = requests.get(f"{_DK_BASE}/categories", headers=_HEADERS, timeout=15)
+        r.raise_for_status()
+        for cat in r.json().get("eventGroupSubcategories", []):
+            cat_name = str(cat.get("name", "") or "").lower()
+            if "batter" in cat_name or "player" in cat_name:
+                cat_id = cat.get("categoryId")
+                for sub in cat.get("subcategories", []):
+                    sub_name = str(sub.get("name", "") or "").lower()
+                    if "home run" in sub_name or "anytime hr" in sub_name:
+                        return cat_id, sub.get("subcategoryId")
+        return None, None
+
+    def _parse_dk(self, data: dict) -> pd.DataFrame:
+        rows: list[dict[str, Any]] = []
+        event_group = data.get("eventGroup", {})
+        for offer_cat in event_group.get("offerCategories", []):
+            for sub_desc in offer_cat.get("offerSubcategoryDescriptors", []):
+                for offer_group in sub_desc.get("offerGroups", []):
+                    event_id = str(offer_group.get("eventId", "") or "")
+                    event_desc = str(offer_group.get("eventDescription", "") or "")
+                    parts = [p.strip() for p in event_desc.replace(" vs ", " @ ").split(" @ ")]
+                    away_team = parts[0] if len(parts) >= 2 else ""
+                    home_team = parts[1] if len(parts) >= 2 else ""
+                    commence_time = str(offer_group.get("startDate", "") or "")
+                    for offer_list in offer_group.get("offers", []):
+                        for offer in (offer_list if isinstance(offer_list, list) else [offer_list]):
+                            player_name_raw = str(offer.get("label", "") or "").strip()
+                            if not player_name_raw:
+                                continue
+                            for outcome in offer.get("outcomes", []):
+                                price_str = str(outcome.get("oddsAmerican", "") or "").replace("+", "")
+                                if not price_str:
+                                    continue
+                                try:
+                                    price = int(price_str)
+                                except ValueError:
+                                    continue
+                                rows.append(_make_row(
+                                    self.provider_name, event_id, commence_time,
+                                    away_team, home_team,
+                                    "DraftKings", "draftkings",
+                                    player_name_raw, price,
+                                ))
+        return pd.DataFrame(rows)
+
+    # ---------------------------------------------------------------------------
+    # FanDuel
+    # ---------------------------------------------------------------------------
+
+    def _fetch_fanduel(self) -> pd.DataFrame:
+        url = (
+            f"{_FD_BASE}/content-managed-page"
+            f"?page=SPORT_LEAGUE&countryCode=US&regionCode=IL"
+            f"&channel=BASEBALL&lang=en-US&_ak={_FD_AK}"
+        )
+        r = requests.get(url, headers=_HEADERS, timeout=20)
+        _log.warning("[fd_scrape] HTTP %s", r.status_code)
+        r.raise_for_status()
+        return self._parse_fd(r.json())
+
+    def _parse_fd(self, data: dict) -> pd.DataFrame:
+        rows: list[dict[str, Any]] = []
+        attachments = data.get("attachments", {})
+        events = attachments.get("events", {})
+        markets = attachments.get("markets", {})
+        for _market_id, market in markets.items():
+            market_type = str(market.get("marketType", "") or "").upper()
+            if "HOME_RUN" not in market_type and "HR" not in market_type:
+                continue
+            event_id = str(market.get("eventId", "") or "")
+            event = events.get(str(event_id), {})
+            away_team = str(
+                event.get("awayTeam", {}).get("name", "")
+                or event.get("awayTeamName", "")
+                or ""
+            )
+            home_team = str(
+                event.get("homeTeam", {}).get("name", "")
+                or event.get("homeTeamName", "")
+                or ""
+            )
+            commence_time = str(event.get("openDate", "") or "")
+            for runner in market.get("runners", []):
+                player_name_raw = str(runner.get("runnerName", "") or "").strip()
+                if not player_name_raw:
+                    continue
+                win_odds = (
+                    runner.get("winRunnerOdds", {})
+                    .get("americanDisplayOdds", {})
+                    .get("americanOdds", "")
+                )
+                if not win_odds:
+                    continue
+                try:
+                    price = int(str(win_odds).replace("+", ""))
+                except ValueError:
+                    continue
+                rows.append(_make_row(
+                    self.provider_name, event_id, commence_time,
+                    away_team, home_team,
+                    "FanDuel", "fanduel",
+                    player_name_raw, price,
+                ))
+        return pd.DataFrame(rows)
+
+    # ---------------------------------------------------------------------------
+    # BetMGM
+    # ---------------------------------------------------------------------------
+
+    def _fetch_betmgm(self) -> pd.DataFrame:
+        url = (
+            "https://sports.nj.betmgm.com/en/sports/api/v2/leagues/baseball-mlb/events"
+            f"?lang=en-us&x-bwin-accessid={_BETMGM_ACCESS_ID}"
+        )
+        r = requests.get(url, headers=_HEADERS, timeout=20)
+        _log.warning("[betmgm_scrape] HTTP %s", r.status_code)
+        r.raise_for_status()
+        return self._parse_betmgm(r.json())
+
+    def _parse_betmgm(self, data: dict | list) -> pd.DataFrame:
+        rows: list[dict[str, Any]] = []
+        events = (
+            data
+            if isinstance(data, list)
+            else data.get("result", {}).get("dataList", data.get("events", []))
+        )
+        for event in events:
+            event_id = str(event.get("id", "") or "")
+            name_obj = event.get("name", {})
+            name = str(name_obj.get("value", "") if isinstance(name_obj, dict) else name_obj or "")
+            parts = [p.strip() for p in name.replace(" vs ", " @ ").split(" @ ")]
+            away_team = parts[0] if len(parts) >= 2 else ""
+            home_team = parts[1] if len(parts) >= 2 else ""
+            commence_time = str(event.get("startDate", "") or "")
+            for fixture in event.get("markets", []):
+                mkt_name_obj = fixture.get("name", {})
+                mkt_name = str(
+                    mkt_name_obj.get("value", "") if isinstance(mkt_name_obj, dict) else mkt_name_obj or ""
+                ).lower()
+                if "home run" not in mkt_name and "hr" not in mkt_name:
+                    continue
+                for selection in fixture.get("selections", []):
+                    sel_name_obj = selection.get("name", {})
+                    player_name_raw = str(
+                        sel_name_obj.get("value", "") if isinstance(sel_name_obj, dict) else sel_name_obj or ""
+                    ).strip()
+                    if not player_name_raw:
+                        continue
+                    price_val = selection.get("price", {}).get("americanOdds")
+                    if price_val is None:
+                        continue
+                    try:
+                        price = int(price_val)
+                    except (TypeError, ValueError):
+                        continue
+                    rows.append(_make_row(
+                        self.provider_name, event_id, commence_time,
+                        away_team, home_team,
+                        "BetMGM", "betmgm",
+                        player_name_raw, price,
+                    ))
+        return pd.DataFrame(rows)
+
+    # ---------------------------------------------------------------------------
+    # Caesars
+    # ---------------------------------------------------------------------------
+
+    def _fetch_caesars(self) -> pd.DataFrame:
+        url = (
+            "https://api.levelmgr.caesarssportsbook.com/api/v1"
+            "/leagues/baseball-mlb/player-props/home-run"
+        )
+        r = requests.get(url, headers=_HEADERS, timeout=20)
+        _log.warning("[caesars_scrape] HTTP %s", r.status_code)
+        r.raise_for_status()
+        return self._parse_caesars(r.json())
+
+    def _parse_caesars(self, data: dict | list) -> pd.DataFrame:
+        rows: list[dict[str, Any]] = []
+        items = (
+            data
+            if isinstance(data, list)
+            else data.get("data", data.get("events", data.get("items", [])))
+        )
+        for item in items:
+            event_id = str(item.get("eventId", item.get("id", "")) or "")
+            away_team = str(item.get("awayTeamName", item.get("away_team", "")) or "")
+            home_team = str(item.get("homeTeamName", item.get("home_team", "")) or "")
+            commence_time = str(item.get("eventDate", item.get("startTime", "")) or "")
+            selections = item.get("participants", item.get("props", item.get("selections", [])))
+            for prop in selections:
+                player_name_raw = str(
+                    prop.get("name", prop.get("participantName", prop.get("playerName", ""))) or ""
+                ).strip()
+                if not player_name_raw:
+                    continue
+                price_val = (
+                    prop.get("odds", {}).get("american")
+                    if isinstance(prop.get("odds"), dict)
+                    else prop.get("americanOdds", prop.get("price"))
+                )
+                if price_val is None:
+                    continue
+                try:
+                    price = int(str(price_val).replace("+", ""))
+                except (TypeError, ValueError):
+                    continue
+                rows.append(_make_row(
+                    self.provider_name, event_id, commence_time,
+                    away_team, home_team,
+                    "Caesars", "williamhill_us",
+                    player_name_raw, price,
+                ))
+        return pd.DataFrame(rows)

data/provider_theoddsapi.py

@@ -309,6 +309,7 @@ class TheOddsAPIProvider(MarketProviderBase):
             odds_params = {
                 "apiKey": ODDS_API_KEY,
                 "regions": "us",
+                "markets": "batter_home_runs",
                 "oddsFormat": "american",
                 "dateFormat": "iso",
             }
@@ -367,17 +368,6 @@ class TheOddsAPIProvider(MarketProviderBase):
                 away_team, home_team, _data_type, _top_keys, _bk_keys,
                 _markets_by_book, _outcome_counts,
             )
-            if _events_attempted == 1:
-                _all_markets = sorted({
-                    m.get("key") for b in bookmakers for m in b.get("markets", [])
-                })
-                _diag_log.warning(
-                    "[upcoming_hr_props] DIAGNOSTIC first_event=%s@%s books_returned=%s markets_available=%s",
-                    away_team, home_team,
-                    [b.get("key") for b in bookmakers],
-                    _all_markets,
-                )
-
             _bookmakers_seen = 0
             _markets_seen = 0
             _outcomes_seen = 0