fix score trend

README.md

@@ -47,5 +47,9 @@ qualification and full-evaluation state for the original-model baseline and each
 miner. The public dashboard shows only hotkeys and stage states; prompts,
 answers, and rejection reasons are excluded.
 
-After a validator restart, the dashboard uses the newest run containing valid
-miner scores. A newer empty run no longer hides an older completed evaluation.
+The dashboard only reads W&B runs whose run state is `running`; stopped, crashed,
+failed, or finished runs are ignored.
+
+Among running validator runs, the dashboard uses the newest run containing valid
+miner scores. A newer empty running run no longer hides an older scored running
+run.

scripts/build_data.py

@@ -129,11 +129,110 @@ def extract_progress(summary: dict) -> dict | None:
     }
 
 
+def _parse_iso(value) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def _snapshot_for_epoch(raw, epoch: float) -> dict | None:
+    if isinstance(raw, str):
+        try:
+            raw = json.loads(raw)
+        except json.JSONDecodeError:
+            return None
+    if not isinstance(raw, dict):
+        return None
+    row_epoch = _as_float(raw.get("epoch"))
+    if row_epoch is None or row_epoch != epoch:
+        return None
+    return raw
+
+
+def extract_stage_timings(run, epoch) -> dict[str, dict]:
+    """Replay a run's progress history to time how long each stage's baseline
+    evaluation actually took for the given epoch.
+
+    The live progress snapshot (summary) only ever holds the *latest* stage
+    state, so a fast baseline/miner pass can flip from "evaluating" to
+    "finished" between two dashboard polls without ever being observed live.
+    Scanning history lets the dashboard show concrete proof ("base done in
+    4.2s") instead of silently losing that transition.
+    """
+    epoch_value = _as_float(epoch)
+    if epoch_value is None:
+        return {}
+
+    active_stage: dict[str, datetime] = {}
+    active_baseline: dict[str, datetime] = {}
+    timings: dict[str, dict] = {}
+
+    try:
+        rows = run.scan_history()
+    except Exception:
+        return {}
+
+    for row in rows:
+        snapshot = _snapshot_for_epoch(row.get("progress/snapshot"), epoch_value)
+        if snapshot is None:
+            continue
+        ts = _parse_iso(snapshot.get("updated_at"))
+        raw_stages = snapshot.get("stages")
+        if not isinstance(raw_stages, dict):
+            continue
+        for stage_name in _PROGRESS_STAGES:
+            stage = raw_stages.get(stage_name)
+            if not isinstance(stage, dict):
+                continue
+            status = stage.get("status")
+            if status == "evaluating" and stage_name not in active_stage:
+                active_stage[stage_name] = ts
+            elif (
+                status in {"completed", "skipped", "failed"}
+                and stage_name in active_stage
+                and "duration_seconds" not in timings.get(stage_name, {})
+            ):
+                start = active_stage[stage_name]
+                if start is not None and ts is not None:
+                    timings.setdefault(stage_name, {})["duration_seconds"] = (
+                        ts - start
+                    ).total_seconds()
+
+            baseline = stage.get("baseline")
+            if baseline == "evaluating" and stage_name not in active_baseline:
+                active_baseline[stage_name] = ts
+            elif (
+                baseline in {"finished", "skipped", "failed"}
+                and stage_name in active_baseline
+                and "baseline_seconds" not in timings.get(stage_name, {})
+            ):
+                start = active_baseline[stage_name]
+                if start is not None and ts is not None:
+                    timings.setdefault(stage_name, {})["baseline_seconds"] = (
+                        ts - start
+                    ).total_seconds()
+
+    return timings
+
+
 def progress_for_run(run, summary: dict) -> dict | None:
     progress = extract_progress(summary)
-    if progress is not None and progress["status"] == "running":
-        if getattr(run, "state", None) in {"crashed", "failed", "killed"}:
-            progress["status"] = "failed"
+    if progress is None:
+        return None
+    if progress["status"] == "running" and getattr(run, "state", None) in {
+        "crashed",
+        "failed",
+        "killed",
+    }:
+        progress["status"] = "failed"
+    timings = extract_stage_timings(run, progress["epoch"])
+    for stage_name, timing in timings.items():
+        stage = progress["stages"].get(stage_name)
+        if isinstance(stage, dict):
+            stage.update(timing)
     return progress
 
 
@@ -147,32 +246,36 @@ def extract_history(run) -> dict:
         epoch = _as_float(row.get("epoch"))
         if epoch is None:
             continue
-        bucket = rows_by_epoch.setdefault(epoch, {})
+        row_metrics = {}
         for key, value in row.items():
             hk = _miner_hotkey_from_score_key(key)
             if hk is not None:
-                bucket[("miner", hk, "score")] = _as_float(value)
+                row_metrics[("miner", hk, "score")] = _as_float(value)
                 miner_keys.add(hk)
             elif key.startswith(_SCORE_PREFIX) and key.endswith(_LEN_SUFFIX):
                 hk2 = key[len(_SCORE_PREFIX) : -len(_LEN_SUFFIX)]
                 if hk2:
-                    bucket[("miner", hk2, "completion_len")] = _as_float(value)
+                    row_metrics[("miner", hk2, "completion_len")] = _as_float(value)
                     miner_keys.add(hk2)
             elif key.startswith(_SCORE_PREFIX) and key.endswith(_CORRECTNESS_SUFFIX):
                 hk2 = key[len(_SCORE_PREFIX) : -len(_CORRECTNESS_SUFFIX)]
                 if hk2:
-                    bucket[("miner", hk2, "correctness_score")] = _as_float(value)
+                    row_metrics[("miner", hk2, "correctness_score")] = _as_float(value)
                     miner_keys.add(hk2)
             elif key == _ORIG_SCORE:
-                bucket[("orig", "score")] = _as_float(value)
-                bucket.setdefault(("orig", "correctness_score"), _score_to_signed_correctness(value))
+                row_metrics[("orig", "score")] = _as_float(value)
+                row_metrics.setdefault(
+                    ("orig", "correctness_score"), _score_to_signed_correctness(value)
+                )
                 saw_orig_score = True
             elif key == _ORIG_CORRECTNESS:
-                bucket[("orig", "correctness_score")] = _as_float(value)
+                row_metrics[("orig", "correctness_score")] = _as_float(value)
                 saw_orig_score = True
             elif key == _ORIG_LEN:
-                bucket[("orig", "completion_len")] = _as_float(value)
+                row_metrics[("orig", "completion_len")] = _as_float(value)
                 saw_orig_len = True
+        if any(value is not None for value in row_metrics.values()):
+            rows_by_epoch.setdefault(epoch, {}).update(row_metrics)
 
     epochs = sorted(rows_by_epoch)
     miners: dict[str, dict] = {}
@@ -300,6 +403,10 @@ def wandb_path(project: str, entity: str | None) -> str:
     return f"{entity}/{project}" if entity else project
 
 
+def _is_running_run(run) -> bool:
+    return getattr(run, "state", None) == "running"
+
+
 def build(project: str, entity: str | None, api=None) -> dict:
     if api is None:
         import wandb
@@ -311,6 +418,8 @@ def build(project: str, entity: str | None, api=None) -> dict:
     fallbacks: dict[str, dict] = {}
     selected: dict[str, dict] = {}
     for run in api.runs(path, order="-created_at"):
+        if not _is_running_run(run):
+            continue
         hotkey = dict(run.config).get("validator_hotkey")
         if not isinstance(hotkey, str) or not hotkey or hotkey in selected:
             continue
@@ -350,6 +459,8 @@ def build_progress(
     seen = set()
     runs = api.runs(path, order="-created_at")
     for run in islice(runs, max(1, run_limit)):
+        if not _is_running_run(run):
+            continue
         hotkey = dict(run.config).get("validator_hotkey")
         if not isinstance(hotkey, str) or not hotkey or hotkey in seen:
             continue

server.py

@@ -70,6 +70,40 @@ def _progress_from_payload(payload: dict) -> dict[str, dict]:
     }
 
 
+def _has_value(values, index: int) -> bool:
+    if not isinstance(values, list) or index >= len(values):
+        return False
+    value = values[index]
+    return isinstance(value, (int, float)) and value == value
+
+
+def _history_has_scored_epoch(validator: dict, epoch) -> bool:
+    history = validator.get("history", {})
+    epochs = history.get("epochs", [])
+    if not isinstance(history, dict) or not isinstance(epochs, list):
+        return False
+    try:
+        index = epochs.index(epoch)
+    except ValueError:
+        return False
+
+    original = history.get("original", {})
+    if isinstance(original, dict):
+        for field in ("correctness_score", "completion_len", "score"):
+            if _has_value(original.get(field), index):
+                return True
+
+    miners = history.get("miners", {})
+    if isinstance(miners, dict):
+        for series in miners.values():
+            if not isinstance(series, dict):
+                continue
+            for field in ("correctness_score", "completion_len", "score"):
+                if _has_value(series.get(field), index):
+                    return True
+    return False
+
+
 def _progress_requires_data_refresh(
     payload: dict, progress_by_validator: dict[str, dict]
 ) -> bool:
@@ -91,6 +125,10 @@ def _progress_requires_data_refresh(
             history_epoch is None or progress_epoch > history_epoch
         ):
             return True
+        if isinstance(progress_epoch, (int, float)) and not _history_has_scored_epoch(
+            validator, progress_epoch
+        ):
+            return True
     return False