Add Track 3 (forecasting) leaderboard section: live skill/rank/fair-skill vs seasonal_naive; pin openmhc@991caf5

Dockerfile

@@ -14,13 +14,15 @@ RUN pip install --no-cache-dir -r requirements.txt
 RUN apt-get update && apt-get install -y --no-install-recommends git \
     && rm -rf /var/lib/apt/lists/*
 
-# Canonical point-flow reducers (skill / rank / fair skill). Installed WITHOUT
+# Canonical point-flow reducers (skill / rank / fair skill) for both the
+# imputation (Track 2) and forecasting (Track 3) leaderboards. Installed WITHOUT
 # deps so the heavy base deps (torch, xgboost, ...) are skipped — only the slim
 # runtime set in requirements.txt is used. Pinned to a commit (current main
 # HEAD) for reproducibility + reliable Docker-layer cache busting; bump this
-# when the reducers change. main@bd5a102 carries the MAPD fair-skill reducer.
+# when the reducers change. main@991caf5 adds the unified forecasting per-user
+# substrate reducers (PR #35).
 RUN pip install --no-cache-dir --no-deps \
-    "openmhc @ git+https://github.com/AshleyLab/myheartcounts-dataset@bd5a1028a19e4ec2eb6965853c538fad71188558"
+    "openmhc @ git+https://github.com/AshleyLab/myheartcounts-dataset@991caf54fc728b8f0b53873f3bc589ac3cb3c1bd"
 
 COPY app.py leaderboard_compute.py logo.png ./
 

app.py

@@ -18,15 +18,18 @@ from pathlib import Path
 from fastapi import FastAPI
 from fastapi.responses import FileResponse, HTMLResponse
 
-from leaderboard_compute import compute_imputation_rows
+from leaderboard_compute import compute_forecasting_rows, compute_imputation_rows
 
 app = FastAPI(title="OpenMHC Leaderboard", docs_url=None, redoc_url=None)
 
-# Compute is mildly expensive (download + reduce); cache the result in-process.
-_CACHE: dict = {"rows": None, "error": None}
+# Compute is mildly expensive (download + reduce); cache per track in-process.
+_CACHE: dict = {
+    "imputation": {"rows": None, "error": None},
+    "forecasting": {"rows": None, "error": None},
+}
 
-# (row key, header label). Order + arrows mirror the paper table.
-COLUMNS = [
+# (row key, header label). Order + arrows mirror the paper tables.
+_IMPUTATION_COLUMNS = [
     ("method", "Method"),
     ("mtype", "Type"),
     ("rank", "R ↓"),
@@ -39,12 +42,42 @@ COLUMNS = [
     ("semantic", "Semantic ↑"),
     ("submitter", "Submitter"),
 ]
-NCOLS = len(COLUMNS)
-
-# sub-track key -> section heading
-SUBTRACKS = [
-    ("single-day", "Single-day imputation"),
-    ("long-context", "Long-context imputation (≥ 7×1440 time steps)"),
+# Forecasting has the same shape minus the imputation-only "semantic" scope.
+_FORECASTING_COLUMNS = [c for c in _IMPUTATION_COLUMNS if c[0] != "semantic"]
+
+_IMPUTATION_NOTE = (
+    "Skill (S) & fairness (S_fair) are % error reduction vs the LOCF baseline "
+    "(paired per-user geometric mean); R is the mean cross-method rank (lower is "
+    "better). Computed live from <code>MyHeartCounts/OpenMHC-leaderboard-data</code>."
+)
+_FORECASTING_NOTE = (
+    "Skill (S) &amp; fairness (S_fair) are % error reduction vs the Seasonal Naive "
+    "baseline (paired per-user geometric mean, category-balanced); R is the mean "
+    "cross-method rank (lower is better). Computed live from "
+    "<code>MyHeartCounts/OpenMHC-leaderboard-data</code>."
+)
+
+# Ordered leaderboard sections rendered on the page (one table each).
+TRACKS = [
+    {
+        "key": "imputation",
+        "compute": compute_imputation_rows,
+        "columns": _IMPUTATION_COLUMNS,
+        "subtracks": [
+            ("single-day", "Single-day imputation"),
+            ("long-context", "Long-context imputation (≥ 7×1440 time steps)"),
+        ],
+        "title": "Track 2a · Imputation",
+        "note": _IMPUTATION_NOTE,
+    },
+    {
+        "key": "forecasting",
+        "compute": compute_forecasting_rows,
+        "columns": _FORECASTING_COLUMNS,
+        "subtracks": [],
+        "title": "Track 3 · Forecasting",
+        "note": _FORECASTING_NOTE,
+    },
 ]
 
 CODE_URL = "https://github.com/AshleyLab/myheartcounts-dataset"
@@ -141,8 +174,8 @@ code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.9em;colo
 
   <div class="hero">
     <p class="eyebrow">OpenMHC Benchmark</p>
-    <h1>Imputation Leaderboard</h1>
-    <p class="lede">Track 2a &mdash; consists of two parts. (1) Reconstructing masked daily, minute-level mobile/wearable signals. (2) Reconstructing masked daily, minute-level mobile/wearable signals given all other days a user has available (neural methods were restricted to 7 days for computational reasons &mdash; 1440&times;19&times;7 is a lot). Ranked by skill score vs the LOCF (last observation carried forward) baseline, computed live from per-user evaluation substrate.</p>
+    <h1>OpenMHC Leaderboard</h1>
+    <p class="lede">Wearable &amp; mobile health benchmark on MyHeartCounts. Track 2a (imputation) reconstructs masked daily, minute-level signals; Track 3 (forecasting) predicts future hourly signals. Each method is ranked by skill score vs a track baseline, computed live from the per-user evaluation substrate.</p>
     <div class="pills">
       <a class="pill" href="%%CODE%%">&#9881;&#65039; Code</a>
       <span class="pill disabled">&#128202; Dataset &middot; coming soon</span>
@@ -152,76 +185,74 @@ code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.9em;colo
     </div>
   </div>
 
-  <div class="section">
-    <p class="section-title">Track 2a &middot; Imputation</p>
-    %%CONTENT%%
-    <p class="note">Skill (S) &amp; fairness (S_fair) are % error reduction vs the LOCF baseline (paired per-user geometric mean); R is the mean cross-method rank (lower is better). Computed live from <code>MyHeartCounts/OpenMHC-leaderboard-data</code>.</p>
-  </div>
+  %%CONTENT%%
 </div>
 <script>
 (function(){
-  var table=document.querySelector('table');
-  if(!table) return;
-  var ths=table.querySelectorAll('thead th');
-  var cur={i:-1,d:1};
   function cell(td){
     if('v' in td.dataset){var n=parseFloat(td.dataset.v);return {num:true,v:isNaN(n)?null:n};}
     return {num:false,v:td.textContent.trim().toLowerCase()};
   }
-  function sortBy(i,d){
-    table.querySelectorAll('tbody').forEach(function(tb){
-      var rows=[].slice.call(tb.querySelectorAll('tr.mrow'));
-      rows.sort(function(a,b){
-        var A=cell(a.children[i]),B=cell(b.children[i]);
-        if(A.num){
-          if(A.v===null&&B.v===null)return 0;
-          if(A.v===null)return 1; if(B.v===null)return -1;
-          return (A.v-B.v)*d;
-        }
-        return String(A.v).localeCompare(String(B.v))*d;
+  // Each table (one per track) gets its own independent sort + Type filter.
+  document.querySelectorAll('table').forEach(function(table){
+    var ths=table.querySelectorAll('thead th');
+    var cur={i:-1,d:1};
+    function sortBy(i,d){
+      table.querySelectorAll('tbody').forEach(function(tb){
+        var rows=[].slice.call(tb.querySelectorAll('tr.mrow'));
+        rows.sort(function(a,b){
+          var A=cell(a.children[i]),B=cell(b.children[i]);
+          if(A.num){
+            if(A.v===null&&B.v===null)return 0;
+            if(A.v===null)return 1; if(B.v===null)return -1;
+            return (A.v-B.v)*d;
+          }
+          return String(A.v).localeCompare(String(B.v))*d;
+        });
+        rows.forEach(function(r){tb.appendChild(r);});
       });
-      rows.forEach(function(r){tb.appendChild(r);});
-    });
-  }
-
-  // Type column: checkbox filter (does NOT sort). Others: click to sort.
-  var boxes=table.querySelectorAll('input.tfilter');
-  function applyFilter(){
-    var checked={}, any=false;
-    boxes.forEach(function(cb){ if(cb.checked){checked[cb.value]=1; any=true;} });
-    var partial=any && checked && (Object.keys(checked).length < boxes.length);
-    document.querySelectorAll('tr.mrow').forEach(function(r){
-      r.style.display=(!any||checked[r.dataset.type])?'':'none';
-    });
-    table.querySelectorAll('tbody').forEach(function(tb){
-      var vis=[].some.call(tb.querySelectorAll('tr.mrow'),function(r){return r.style.display!=='none';});
-      var sec=tb.querySelector('tr.sec'); if(sec) sec.style.display=vis?'':'none';
-    });
-    table.querySelectorAll('.filter-th').forEach(function(t){t.classList.toggle('active', partial);});
-  }
-  boxes.forEach(function(cb){ cb.addEventListener('change', applyFilter); });
+    }
 
-  ths.forEach(function(th,i){
-    if(th.dataset.filter){  // Type = filter dropdown, not sortable
-      th.addEventListener('click',function(e){
-        if(e.target.closest('.dropdown')) return;  // clicks inside menu don't toggle
-        th.classList.toggle('open');
+    // Type column: checkbox filter (does NOT sort). Others: click to sort.
+    var boxes=table.querySelectorAll('input.tfilter');
+    function applyFilter(){
+      var checked={}, any=false;
+      boxes.forEach(function(cb){ if(cb.checked){checked[cb.value]=1; any=true;} });
+      var partial=any && (Object.keys(checked).length < boxes.length);
+      table.querySelectorAll('tr.mrow').forEach(function(r){
+        r.style.display=(!any||checked[r.dataset.type])?'':'none';
+      });
+      table.querySelectorAll('tbody').forEach(function(tb){
+        var vis=[].some.call(tb.querySelectorAll('tr.mrow'),function(r){return r.style.display!=='none';});
+        var sec=tb.querySelector('tr.sec'); if(sec) sec.style.display=vis?'':'none';
       });
-      return;
+      table.querySelectorAll('.filter-th').forEach(function(t){t.classList.toggle('active', partial);});
     }
-    th.classList.add('sortable');
-    th.addEventListener('click',function(){
-      var d=(cur.i===i)?-cur.d:(th.dataset.def==='asc'?1:-1);
-      cur={i:i,d:d};
-      ths.forEach(function(h){h.classList.remove('s-asc','s-desc');});
-      th.classList.add(d===1?'s-asc':'s-desc');
-      sortBy(i,d);
+    boxes.forEach(function(cb){ cb.addEventListener('change', applyFilter); });
+
+    ths.forEach(function(th,i){
+      if(th.dataset.filter){  // Type = filter dropdown, not sortable
+        th.addEventListener('click',function(e){
+          if(e.target.closest('.dropdown')) return;  // clicks inside menu don't toggle
+          th.classList.toggle('open');
+        });
+        return;
+      }
+      th.classList.add('sortable');
+      th.addEventListener('click',function(){
+        var d=(cur.i===i)?-cur.d:(th.dataset.def==='asc'?1:-1);
+        cur={i:i,d:d};
+        ths.forEach(function(h){h.classList.remove('s-asc','s-desc');});
+        th.classList.add(d===1?'s-asc':'s-desc');
+        sortBy(i,d);
+      });
     });
   });
 
+  // One global handler closes any open Type dropdown across all tables.
   document.addEventListener('click',function(e){
     if(!e.target.closest('.filter-th')){
-      table.querySelectorAll('.filter-th.open').forEach(function(t){t.classList.remove('open');});
+      document.querySelectorAll('.filter-th.open').forEach(function(t){t.classList.remove('open');});
     }
   });
 })();
@@ -231,13 +262,14 @@ code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.9em;colo
 """
 
 
-def _rows() -> tuple[list[dict] | None, str | None]:
-    if _CACHE["rows"] is None and _CACHE["error"] is None:
+def _rows(track_key: str, compute) -> tuple[list[dict] | None, str | None]:
+    cache = _CACHE[track_key]
+    if cache["rows"] is None and cache["error"] is None:
         try:
-            _CACHE["rows"] = compute_imputation_rows()
+            cache["rows"] = compute()
         except Exception as e:  # surface the failure on the page; don't hide it
-            _CACHE["error"] = f"{type(e).__name__}: {e}"
-    return _CACHE["rows"], _CACHE["error"]
+            cache["error"] = f"{type(e).__name__}: {e}"
+    return cache["rows"], cache["error"]
 
 
 # Per-column heatmap (paper-style blue gradient): better -> more saturated blue.
@@ -297,9 +329,9 @@ def _skill_key(r: dict):
     return (s is None, -(s if s is not None else 0.0))
 
 
-def _method_row(r: dict, section_key: str, stats: dict) -> str:
+def _method_row(r: dict, section_key: str, stats: dict, columns: list) -> str:
     cells = []
-    for k, _ in COLUMNS:
+    for k, _ in columns:
         if k == "method":
             name = html.escape(str(r.get(k)))
             url = r.get("model_url")
@@ -335,12 +367,21 @@ def _th(key: str, label: str, type_values: list[str]) -> str:
     return f'<th data-def="{default}">{html.escape(label)}</th>'
 
 
-def _table(rows: list[dict]) -> str:
+def _table(rows: list[dict], columns: list, subtracks: list) -> str:
+    ncols = len(columns)
     type_values = sorted({str(r["mtype"]) for r in rows if r.get("mtype")})
-    head = "".join(_th(k, label, type_values) for k, label in COLUMNS)
+    head = "".join(_th(k, label, type_values) for k, label in columns)
     stats = _col_stats(rows)
-    known = {k for k, _ in SUBTRACKS}
-    sections = list(SUBTRACKS)
+
+    if not subtracks:
+        # Single ungrouped table (no sub-track headers).
+        sub_rows = sorted(rows, key=_skill_key)  # default order; JS re-sorts on click
+        mrows = "".join(_method_row(r, "all", stats, columns) for r in sub_rows)
+        tbodies = f'<tbody data-subtrack="all">{mrows}</tbody>'
+        return f'<div class="card"><table><thead><tr>{head}</tr></thead>{tbodies}</table></div>'
+
+    known = {k for k, _ in subtracks}
+    sections = list(subtracks)
     if any(r.get("subtrack") not in known for r in rows):
         sections = sections + [("other", "Other")]
 
@@ -353,23 +394,34 @@ def _table(rows: list[dict]) -> str:
         if not sub_rows:
             continue
         sub_rows = sorted(sub_rows, key=_skill_key)  # default order; JS re-sorts on click
-        sec = f'<tr class="sec"><td colspan="{NCOLS}">{html.escape(label)}</td></tr>'
-        mrows = "".join(_method_row(r, key, stats) for r in sub_rows)
+        sec = f'<tr class="sec"><td colspan="{ncols}">{html.escape(label)}</td></tr>'
+        mrows = "".join(_method_row(r, key, stats, columns) for r in sub_rows)
         tbodies += f'<tbody data-subtrack="{html.escape(key)}">{sec}{mrows}</tbody>'
 
     table = f'<div class="card"><table><thead><tr>{head}</tr></thead>{tbodies}</table></div>'
     return table
 
 
-@app.get("/", response_class=HTMLResponse)
-def index() -> str:
-    rows, error = _rows()
+def _render_section(cfg: dict) -> str:
+    rows, error = _rows(cfg["key"], cfg["compute"])
     if error is not None:
-        content = f'<div class="err">Failed to compute leaderboard:\n{html.escape(error)}</div>'
+        body = f'<div class="err">Failed to compute leaderboard:\n{html.escape(error)}</div>'
     elif not rows:
-        content = '<div class="err">No methods found in the substrate dataset.</div>'
+        body = '<div class="err">No methods found in the substrate dataset.</div>'
     else:
-        content = _table(rows)
+        body = _table(rows, cfg["columns"], cfg["subtracks"])
+    return (
+        '<div class="section">'
+        f'<p class="section-title">{html.escape(cfg["title"])}</p>'
+        f"{body}"
+        f'<p class="note">{cfg["note"]}</p>'
+        "</div>"
+    )
+
+
+@app.get("/", response_class=HTMLResponse)
+def index() -> str:
+    content = "\n".join(_render_section(cfg) for cfg in TRACKS)
     return (
         PAGE.replace("%%CONTENT%%", content)
         .replace("%%CODE%%", CODE_URL)
@@ -380,8 +432,14 @@ def index() -> str:
 
 @app.get("/health")
 def health() -> dict:
-    rows, error = _rows()
-    return {"status": "ok" if error is None else "error", "methods": len(rows or []), "error": error}
+    tracks: dict = {}
+    ok = True
+    for cfg in TRACKS:
+        rows, error = _rows(cfg["key"], cfg["compute"])
+        tracks[cfg["key"]] = {"methods": len(rows or []), "error": error}
+        if error is not None:
+            ok = False
+    return {"status": "ok" if ok else "error", "tracks": tracks}
 
 
 @app.get("/logo.png")

leaderboard_compute.py

@@ -154,7 +154,129 @@ def compute_imputation_rows() -> list[dict]:
     return rows
 
 
+# ---------------------------------------------------------------------------
+# Track 3 — forecasting
+# ---------------------------------------------------------------------------
+
+FC_BASELINE = "seasonal_naive"
+# Forecasting has no published HF checkpoints wired yet; fill in as they land
+# (keyed by internal method id, e.g. "dlinear": "https://huggingface.co/...").
+FC_MODEL_URLS: dict[str, str] = {}
+
+
+def _load_demographics() -> dict[str, dict[str, str]]:
+    """Build ``{user_id: {age_group, sex}}`` from the imputation substrate.
+
+    The forecasting per-user substrate carries no demographic columns, but every
+    forecasting user is also in the imputation substrate (which bakes per-user
+    ``age_group`` / ``sex`` subgroup rows under the canonical 18/30/40/50/60 age
+    bins). Reusing that partition lets forecasting fairness use the *identical*
+    user->subgroup mapping as imputation fairness, with no extra artifact.
+    """
+    from huggingface_hub import hf_hub_download
+
+    path = hf_hub_download(DATASET_REPO, "imputation/locf.parquet", repo_type="dataset")
+    imp = pd.read_parquet(path, columns=["subgroup_attr", "subgroup_value", "user_id"])
+    demo: dict[str, dict[str, str]] = {}
+    for attr in ("age_group", "sex"):
+        sub = imp[imp["subgroup_attr"].astype(str) == attr][
+            ["user_id", "subgroup_value"]
+        ].drop_duplicates()
+        for uid, val in sub.itertuples(index=False):
+            demo.setdefault(str(uid), {})[attr] = str(val)
+    return demo
+
+
+def compute_forecasting_rows() -> list[dict]:
+    """Return leaderboard rows (sorted by skill desc) for the forecasting track."""
+    from forecasting_evaluation.metrics import metric_spec as spec
+    from forecasting_evaluation.metrics.fair_skill_score import (
+        compute_fair_skill_scores_from_errors,
+    )
+    from forecasting_evaluation.metrics.grouped_metric_rank_summary import (
+        build_grouped_metric_rank_tables,
+    )
+    from forecasting_evaluation.metrics.per_user_errors import to_error_df
+    from forecasting_evaluation.metrics.skill_score_summary import compute_skill_score_tables
+
+    df, root = load_substrate("forecasting")
+    methods = sorted(df["model"].astype(str).unique())
+    models = {m: {"path": "", "display_name": m} for m in methods}
+
+    # Skill: overall (category-balanced) + per-sensor-category scopes, vs the
+    # Seasonal Naive baseline. summary_df is one row per model with *_score cols.
+    _, summary, _ = compute_skill_score_tables(
+        models=models,
+        baseline_model=FC_BASELINE,
+        continuous_metrics=list(spec.PAPER_CONTINUOUS_METRICS),
+        binary_metrics=list(spec.PAPER_BINARY_METRICS),
+        continuous_channel_indices=spec.CONTINUOUS_CHANNELS,
+        binary_channel_indices=spec.BINARY_CHANNELS,
+        clip_lower=spec.PAPER_CLIP_LOWER,
+        clip_upper=spec.PAPER_CLIP_UPPER,
+        min_pairs=spec.PAPER_MIN_PAIRS,
+        aggregation_unit="user",
+        within_user_aggregation="micro",
+        per_user_metrics=df,
+    )
+    summary["model"] = summary["model"].astype(str)
+    summary = summary.set_index("model")
+
+    # Cross-method average rank; the headline is the category-balanced "overall".
+    _, rank_long, _ = build_grouped_metric_rank_tables(
+        models=models,
+        continuous_metrics=list(spec.PAPER_CONTINUOUS_METRICS),
+        binary_metrics=list(spec.PAPER_BINARY_METRICS),
+        continuous_channel_indices=spec.CONTINUOUS_CHANNELS,
+        binary_groups=[(name, tuple(idx)) for name, idx in spec.BINARY_GROUPS],
+        within_user_aggregation="micro",
+        per_user_metrics=df,
+    )
+    rank_long["model"] = rank_long["model"].astype(str)
+    overall_rank = rank_long[rank_long["scope"] == "overall"].set_index("model")["rank"]
+
+    # Disparity-ratio (MAPD) fair skill across age_group + sex. Demographics come
+    # from the imputation substrate (same user->subgroup partition); the reducer
+    # is the canonical forecasting one (point estimate == leaderboard value).
+    err = to_error_df(df, user_col="user_id")
+    fair = compute_fair_skill_scores_from_errors(
+        err, _load_demographics(), baseline_method=FC_BASELINE
+    )
+    fair_overall = fair[fair["scope"] == "overall"].set_index("model")["fair_skill_score"]
+
+    def num(value) -> float | None:
+        if value is None or (isinstance(value, float) and pd.isna(value)):
+            return None
+        return float(value)
+
+    rows = []
+    for m in methods:
+        name, mtype, submitter, subtrack = read_method_meta(root, "forecasting", m)
+        srow = summary.loc[m] if m in summary.index else None
+        rows.append(
+            {
+                "method": name,
+                "mtype": mtype,
+                "submitter": submitter,
+                "subtrack": subtrack,
+                "model_url": FC_MODEL_URLS.get(m),
+                "skill": num(srow["overall_score"]) if srow is not None else None,
+                "fair_skill": num(fair_overall.get(m)),
+                "rank": num(overall_rank.get(m)),
+                "activity": num(srow["activity_score"]) if srow is not None else None,
+                "physiology": num(srow["physiology_score"]) if srow is not None else None,
+                "sleep": num(srow["sleep_score"]) if srow is not None else None,
+                "workout": num(srow["workout_score"]) if srow is not None else None,
+            }
+        )
+    rows.sort(key=lambda r: (r["skill"] is None, -(r["skill"] or 0.0)))
+    return rows
+
+
 if __name__ == "__main__":
     import json
 
-    print(json.dumps(compute_imputation_rows(), indent=2))
+    print(json.dumps({
+        "imputation": compute_imputation_rows(),
+        "forecasting": compute_forecasting_rows(),
+    }, indent=2))