feat: MOVDA raw delta features (Cat37 6→13) + beta calibration + browser scraper

- engine.py: add 7 raw delta_MOV rolling features (no EWM smoothing)
- app.py: beta calibration method (betacal) added to evolution search space
- browser_scraper.py: Crawl4AI + requests fallback for odds scraping
- scrape_odds.py: structured odds scraper
- requirements.txt: betacal, html2text, beautifulsoup4

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Dockerfile.browser

@@ -0,0 +1,64 @@
+# Dockerfile.browser -- Browser-enabled HF Space image for web scraping
+# ======================================================================
+#
+# NOT ACTIVE YET -- this is a template for when we need browser-based
+# scraping on HF Spaces (e.g., scraping odds pages with JS rendering).
+#
+# Current HF Spaces use the default Python runtime without browser deps.
+# To activate: rename to Dockerfile and push to the target Space.
+#
+# Requirements:
+#   - HF Space must be configured as "Docker" SDK (not Gradio SDK)
+#   - The Space will be larger (~2GB) due to Chromium
+#   - CPU-only is fine for scraping (no GPU needed)
+#
+# Size estimate: ~2.5GB image (Playwright + Chromium + Python deps)
+
+FROM python:3.11-slim-bookworm
+
+# Install system deps for Playwright/Chromium
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    ca-certificates \
+    fonts-liberation \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libnspr4 \
+    libnss3 \
+    libx11-xcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxrandr2 \
+    xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user (HF Spaces requirement)
+RUN useradd -m -u 1000 user
+WORKDIR /home/user/app
+
+# Install Python deps
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install Playwright and Chromium browser
+RUN pip install --no-cache-dir crawl4ai>=0.4 playwright
+RUN playwright install chromium
+RUN playwright install-deps chromium
+
+# Copy application code
+COPY . .
+
+# Fix permissions
+RUN chown -R user:user /home/user/app
+
+USER user
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

app.py

@@ -634,7 +634,7 @@ class Individual:
             "reg_alpha": 10 ** random.uniform(-6, 1),
             "reg_lambda": 10 ** random.uniform(-6, 1),
             "model_type": model_type or random.choice(CPU_MODEL_TYPES if not _HAS_GPU else ALL_MODEL_TYPES),
-            "calibration": random.choices(["none", "sigmoid", "venn_abers"], weights=[30, 20, 50], k=1)[0],
+            "calibration": random.choices(["none", "sigmoid", "venn_abers", "beta"], weights=[25, 15, 30, 30], k=1)[0],
             # Neural net hyperparams (used only for NN model types)
             "nn_hidden_dims": random.choice([64, 128, 256]),
             "nn_n_layers": random.randint(2, 4),
@@ -708,7 +708,7 @@ class Individual:
         if random.random() < 0.15: self.hyperparams["max_depth"] = max(2, min(8, self.hyperparams["max_depth"] + random.randint(-2, 2)))
         if random.random() < 0.15: self.hyperparams["learning_rate"] = max(0.001, min(0.3, self.hyperparams["learning_rate"] * 10 ** random.uniform(-0.3, 0.3)))
         if random.random() < 0.08: self.hyperparams["model_type"] = random.choice(CPU_MODEL_TYPES if not _HAS_GPU else ALL_MODEL_TYPES)
-        if random.random() < 0.05: self.hyperparams["calibration"] = random.choices(["none", "sigmoid", "venn_abers"], weights=[60, 20, 20], k=1)[0]
+        if random.random() < 0.05: self.hyperparams["calibration"] = random.choices(["none", "sigmoid", "venn_abers", "beta"], weights=[50, 15, 15, 20], k=1)[0]
         # Neural net hyperparams mutation
         if random.random() < 0.10: self.hyperparams["nn_hidden_dims"] = random.choice([64, 128, 256, 512])
         if random.random() < 0.10: self.hyperparams["nn_n_layers"] = max(1, min(6, self.hyperparams.get("nn_n_layers", 2) + random.randint(-1, 1)))
@@ -1249,10 +1249,12 @@ def evaluate(ind, X, y, n_splits=2, fast=True, eval_counter=[0]):
                 # Purge last PURGE_GAP games from training to avoid temporal leakage
                 ti_safe = ti[:-PURGE_GAP] if len(ti) > PURGE_GAP + 50 else ti
                 m = clone(model)
-                # Calibration: none (default, best empirically), sigmoid (Platt), or venn_abers (MAPIE)
+                # Calibration: none (default), sigmoid (Platt), venn_abers (MAPIE), or beta (BetaCalibration)
                 cal_method = hp_eval.get("calibration", "none")
                 if cal_method == "isotonic":
                     cal_method = "none"  # Isotonic empirically hurts Brier (+0.003 to +0.007)
+                _beta_cal = None   # beta calibrator applied post-predict
+                _model_fitted = False  # tracks whether m.fit() was already called
                 if cal_method == "venn_abers":
                     try:
                         from mapie.classification import MapieClassifier
@@ -1261,13 +1263,30 @@ def evaluate(ind, X, y, n_splits=2, fast=True, eval_counter=[0]):
                         mapie = MapieClassifier(m_inner, method="lac", cv="prefit")
                         mapie.fit(X_sub[ti_safe[-200:]], y_eval[ti_safe[-200:]])
                         m = mapie  # MapieClassifier wraps fitted model
-                        cal_method = "none"  # Skip CalibratedClassifierCV below
+                        _model_fitted = True
+                        cal_method = "none"
                     except (ImportError, Exception):
                         cal_method = "none"  # Fallback if MAPIE not installed
+                if cal_method == "beta":
+                    try:
+                        from betacal import BetaCalibration
+                        # Fit base model, then fit beta calibrator on a held-out slice
+                        m.fit(X_sub[ti_safe], y_eval[ti_safe])
+                        _model_fitted = True
+                        cal_slice = ti_safe[-200:] if len(ti_safe) > 200 else ti_safe
+                        raw_p = m.predict_proba(X_sub[cal_slice])[:, 1]
+                        _beta_cal = BetaCalibration(parameters="abm")
+                        _beta_cal.fit(raw_p.reshape(-1, 1), y_eval[cal_slice])
+                        cal_method = "none"
+                    except (ImportError, Exception):
+                        cal_method = "none"  # Fallback if betacal not installed
                 if cal_method == "sigmoid":
                     m = CalibratedClassifierCV(m, method=cal_method, cv=3)
-                m.fit(X_sub[ti_safe], y_eval[ti_safe])
+                if not _model_fitted:
+                    m.fit(X_sub[ti_safe], y_eval[ti_safe])
                 p = m.predict_proba(X_sub[vi])[:, 1]
+                if _beta_cal is not None:
+                    p = _beta_cal.predict(p.reshape(-1, 1))
                 briers.append(brier_score_loss(y_eval[vi], p))
                 rois.append(_log_loss_score(p, y_eval[vi]))
                 all_p.extend(p); all_y.extend(y_eval[vi])

features/engine.py

@@ -2223,7 +2223,7 @@ class NBAFeatureEngine:
             "dense_sched_x_margin",                   # schedule_density × margin_diff
         ])
 
-        # 37. MOVDA ELO FEATURES (6 features) — arXiv:2506.00348
+        # 37. MOVDA ELO FEATURES (13 features) — arXiv:2506.00348
         # Margin-of-Victory Differential Analysis: R' = R + K*(S-E) + λ*(MOV-E_MOV)
         for prefix in ["h", "a"]:
             names.append(f"{prefix}_movda_rating")          # MOVDA Elo rating (normalized)
@@ -2232,6 +2232,12 @@ class NBAFeatureEngine:
             "movda_diff",                                    # MOVDA rating differential
             "movda_win_prob",                                # MOVDA-derived win probability
         ])
+        # Raw delta_MOV rolling features (no EWM smoothing) — captures recent surprise momentum
+        for prefix in ["h", "a"]:
+            names.append(f"{prefix}_delta_mov_raw")         # last game's raw MOV surprise
+            names.append(f"{prefix}_delta_mov_rolling_5")   # rolling mean over last 5 games
+            names.append(f"{prefix}_delta_mov_rolling_10")  # rolling mean over last 10 games
+        names.append("delta_mov_diff")                       # h_delta_mov_rolling_5 - a_delta_mov_rolling_5
 
         # 38. VENUE-CONDITIONAL MATCHUP FEATURES (14 features)
         # Home team's home-only stats vs away team's road-only stats
@@ -2281,6 +2287,7 @@ class NBAFeatureEngine:
         # ── Category 37: MOVDA ELO state trackers ──
         team_movda = defaultdict(lambda: 1500.0)           # MOVDA Elo rating
         mov_surprise_ewm = defaultdict(float)               # Per-team EWMA of MOV surprise
+        delta_mov_history = defaultdict(list)               # Per-team raw delta_MOV history
         _MOVDA_K = 20.0; _MOVDA_C = 400.0; _MOVDA_LAMBDA = 0.3
         _MOVDA_ALPHA = 19.2511; _MOVDA_BETA = 0.002342
         _MOVDA_GAMMA = 648.0334; _MOVDA_DELTA = -645.8717
@@ -2339,6 +2346,7 @@ class NBAFeatureEngine:
                     team_home_margin_sum, team_home_games_count)
                 # Update MOVDA ELO (Cat 37)
                 self._update_movda(home, away, hs, as_, team_movda, mov_surprise_ewm,
+                                   delta_mov_history,
                                    _MOVDA_K, _MOVDA_C, _MOVDA_LAMBDA, _MOVDA_ALPHA,
                                    _MOVDA_BETA, _MOVDA_GAMMA, _MOVDA_DELTA, _MOVDA_EWM_ALPHA)
                 continue
@@ -5049,7 +5057,7 @@ class NBAFeatureEngine:
                 (self._games_in_window(hr_, gd, 7) - self._games_in_window(ar_, gd, 7)) * (_h_margin5 - _a_margin5),
             ])
 
-            # ── 37. MOVDA ELO FEATURES (6 features) ──
+            # ── 37. MOVDA ELO FEATURES (13 features) ──
             _movda_dr = team_movda[home] - team_movda[away]
             _movda_wp = 1.0 / (1.0 + 10.0 ** (-_movda_dr / _MOVDA_C))
             for _mt, _mk in [(home, home), (away, away)]:
@@ -5057,6 +5065,21 @@ class NBAFeatureEngine:
                 row.append(mov_surprise_ewm[_mk] / 20.0)           # mov_surprise_ewm (normalized)
             row.append(_movda_dr / 400.0)                           # movda_diff
             row.append(_movda_wp)                                    # movda_win_prob
+            # Raw delta_MOV rolling features (no EWM smoothing)
+            for _mk in [home, away]:
+                _dh = delta_mov_history[_mk]
+                _raw = (_dh[-1] / 20.0) if _dh else 0.0
+                _roll5 = (sum(_dh[-5:]) / len(_dh[-5:]) / 20.0) if _dh else 0.0
+                _roll10 = (sum(_dh[-10:]) / len(_dh[-10:]) / 20.0) if _dh else 0.0
+                row.append(_raw)    # {prefix}_delta_mov_raw
+                row.append(_roll5)  # {prefix}_delta_mov_rolling_5
+                row.append(_roll10) # {prefix}_delta_mov_rolling_10
+            # delta_mov_diff: home rolling_5 - away rolling_5
+            _h_dh = delta_mov_history[home]
+            _a_dh = delta_mov_history[away]
+            _h_r5 = (sum(_h_dh[-5:]) / len(_h_dh[-5:]) / 20.0) if _h_dh else 0.0
+            _a_r5 = (sum(_a_dh[-5:]) / len(_a_dh[-5:]) / 20.0) if _a_dh else 0.0
+            row.append(_h_r5 - _a_r5)                               # delta_mov_diff
 
             # ── 38. VENUE-CONDITIONAL MATCHUP FEATURES (14 features) ──
             # Use true venue-specific records: home team at home vs away team on road
@@ -5104,6 +5127,7 @@ class NBAFeatureEngine:
                 team_home_margin_sum, team_home_games_count)
             # Update MOVDA ELO (Cat 37)
             self._update_movda(home, away, hs, as_, team_movda, mov_surprise_ewm,
+                               delta_mov_history,
                                _MOVDA_K, _MOVDA_C, _MOVDA_LAMBDA, _MOVDA_ALPHA,
                                _MOVDA_BETA, _MOVDA_GAMMA, _MOVDA_DELTA, _MOVDA_EWM_ALPHA)
 
@@ -5659,8 +5683,9 @@ class NBAFeatureEngine:
         team_home_games_count[home] += 1
 
     def _update_movda(self, home, away, hs, as_, team_movda, mov_surprise_ewm,
+                      delta_mov_history,
                       K, C, lam, alpha, beta, gamma, delta_param, ewm_alpha):
-        """Update MOVDA Elo ratings (Cat 37). arXiv:2506.00348."""
+        """Update MOVDA Elo ratings and raw delta_MOV history (Cat 37). arXiv:2506.00348."""
         margin = hs - as_
         result = 1.0 if margin > 0 else (0.0 if margin < 0 else 0.5)
         delta_r = team_movda[home] - team_movda[away]
@@ -5672,6 +5697,9 @@ class NBAFeatureEngine:
         team_movda[away] -= movda_update
         mov_surprise_ewm[home] = ewm_alpha * delta_mov + (1 - ewm_alpha) * mov_surprise_ewm[home]
         mov_surprise_ewm[away] = ewm_alpha * (-delta_mov) + (1 - ewm_alpha) * mov_surprise_ewm[away]
+        # Append raw delta_MOV to rolling history (home team's perspective)
+        delta_mov_history[home].append(delta_mov)
+        delta_mov_history[away].append(-delta_mov)
 
     def _parse_stats(self, stats, pts, opp_pts, is_home=True):
         """Extract stats from game data. Uses REAL box score when available, estimates otherwise."""

requirements.txt

@@ -11,4 +11,9 @@ psycopg2-binary>=2.9
 torch>=2.3 --index-url https://download.pytorch.org/whl/cpu
 pytorch_tabnet>=4.1
 mapie>=0.9
+betacal>=0.1
 # autogluon.tabular>=1.2  # OPTIONAL — large install (~2GB), uncomment if needed
+# --- Browser scraping (needs Playwright deps in Docker image, see Dockerfile.browser) ---
+# crawl4ai>=0.4  # OPTIONAL — uncomment when using Dockerfile.browser for browser-based scraping
+html2text>=2024.2  # Lightweight HTML-to-markdown for requests fallback
+beautifulsoup4>=4.12  # CSS selector extraction in requests fallback