| """Official-style row-by-row evaluation without sequence-boundary leakage.""" |
|
|
| from __future__ import annotations |
|
|
| from dataclasses import dataclass |
| from typing import Optional |
|
|
| import numpy as np |
| import pandas as pd |
|
|
| from competition_package.utils import DataPoint |
| from src.data.protocol import get_feature_columns, validate_wunder_dataframe |
| from src.utils.metrics import compute_r2_per_feature, compute_r2_score |
|
|
|
|
| @dataclass(frozen=True) |
| class StepwiseScoreResult: |
| mean_r2: float |
| r2_per_feature: dict[str, float] |
| n_predictions_scored: int |
| n_predictions_requested: int |
| n_predictions_dropped_final_step: int |
| feature_columns: list[str] |
|
|
|
|
| class StepwiseScorer: |
| """Replay held-out sequences through the competition `predict` API.""" |
|
|
| def __init__(self, dataframe: pd.DataFrame): |
| self.dataset = dataframe.sort_values(["seq_ix", "step_in_seq"]).reset_index(drop=True) |
| self.feature_columns = validate_wunder_dataframe(self.dataset) |
| self.dim = len(self.feature_columns) |
|
|
| @classmethod |
| def from_parquet(cls, path: str, seq_ids: Optional[list[int]] = None) -> "StepwiseScorer": |
| df = pd.read_parquet(path) |
| if seq_ids is not None: |
| df = df[df["seq_ix"].isin(set(seq_ids))].copy() |
| return cls(df) |
|
|
| def score(self, model) -> StepwiseScoreResult: |
| predictions = [] |
| targets = [] |
| pending_prediction = None |
| pending_seq_ix = None |
| requested = 0 |
| dropped_final = 0 |
|
|
| for _, seq_df in self.dataset.groupby("seq_ix", sort=True): |
| pending_prediction = None |
| pending_seq_ix = None |
|
|
| seq_ix_values = seq_df["seq_ix"].to_numpy() |
| step_values = seq_df["step_in_seq"].to_numpy() |
| need_values = seq_df["need_prediction"].to_numpy() |
| state_values = seq_df[self.feature_columns].to_numpy(dtype=np.float32) |
|
|
| for pos in range(len(seq_df)): |
| seq_ix = int(seq_ix_values[pos]) |
| step_in_seq = int(step_values[pos]) |
| need_prediction = bool(need_values[pos]) |
| state = state_values[pos] |
|
|
| if pending_prediction is not None: |
| if pending_seq_ix != seq_ix: |
| raise AssertionError("Internal scorer leaked across sequences") |
| predictions.append(pending_prediction) |
| targets.append(state) |
|
|
| data_point = DataPoint(seq_ix, step_in_seq, need_prediction, state) |
| next_prediction = model.predict(data_point) |
| self._check_prediction(data_point, next_prediction) |
| if need_prediction: |
| requested += 1 |
| pending_prediction = next_prediction |
| pending_seq_ix = seq_ix |
|
|
| if pending_prediction is not None: |
| dropped_final += 1 |
|
|
| if not predictions: |
| raise ValueError("No predictions were scored") |
|
|
| y_pred = np.asarray(predictions, dtype=np.float64) |
| y_true = np.asarray(targets, dtype=np.float64) |
| return StepwiseScoreResult( |
| mean_r2=compute_r2_score(y_true, y_pred), |
| r2_per_feature=compute_r2_per_feature(y_true, y_pred, self.feature_columns), |
| n_predictions_scored=len(predictions), |
| n_predictions_requested=requested, |
| n_predictions_dropped_final_step=dropped_final, |
| feature_columns=self.feature_columns, |
| ) |
|
|
| def _check_prediction(self, data_point: DataPoint, prediction: Optional[np.ndarray]) -> None: |
| if not data_point.need_prediction: |
| if prediction is not None: |
| raise ValueError(f"Prediction is not needed for {data_point}") |
| return |
| if prediction is None: |
| raise ValueError(f"Prediction is required for {data_point}") |
| if not isinstance(prediction, np.ndarray): |
| raise ValueError(f"Prediction must be np.ndarray, got {type(prediction)}") |
| if prediction.shape != (self.dim,): |
| raise ValueError(f"Prediction has wrong shape: {prediction.shape} != {(self.dim,)}") |
| if not np.isfinite(prediction).all(): |
| raise ValueError("Prediction contains NaN or Inf") |
|
|
