Deploy resolved perturbation benchmark branch

Source branch: julienduquesne/perturbation-tasks at d9477beab1597762fb3b4f3b695e4ed5dba289cf

README.md

@@ -175,11 +175,13 @@ new datasets until their owners submit embeddings for `d011`–`d016`.
 ## Baselines
 
 `task_results.csv` carries an `is_baseline` flag. Reference submissions we
-produce ourselves (a random embedding, PCA / HVG recipes over log-CPM) are
-published with it set, rendered as `name (baseline)`, and **ranked in place**.
-A foundation model losing to a PCA is exactly the result worth publishing, so we
-keep it in the table rather than tucked underneath. They are generated and
-pushed by `benchmark/public_benchmark/baselines.py --score --publish`.
+produce ourselves are published with it set, rendered as `name (baseline)`, and
+**ranked in place**. A foundation model losing to raw log-CPM expression is
+exactly the result worth publishing, so we keep it in the table rather than
+tucked underneath. Two are published today, both log2(CPM+1) with no gene
+scaling: `HVG-1200-genes` (the 1,200 highest-variance genes) and `Raw data`
+(every gene). The HVG selection is label-blind but fit over every sample of a
+dataset, so read both as a floor set by the raw features.
 
 ## Space configuration
 
@@ -209,7 +211,7 @@ pushed by `benchmark/public_benchmark/baselines.py --score --publish`.
 The three dataset repos are derived from one constant, `ORG` in `evaluator.py`.
 The rest of the org name is spelled out and has to be changed by hand:
 
-- `SPACE_REPO` in `benchmark/public_benchmark/deploy_space.py`
+- `SPACE_REPO` in `deploy_space.py`
 - `PUBLIC_REPO` in `quickstart.py`
 - the links in this file and in `pages/*.md`
 

leaderboard.py

@@ -15,7 +15,7 @@ Two tables per board, deliberately different:
 
 Our own baselines are entries like any other: they are labelled, and they rank
 where their score puts them. A baseline pinned to the bottom would hide the one
-result worth publishing -- a foundation model losing to a PCA.
+result worth publishing: a foundation model losing to the raw expression.
 
 Task metadata is read defensively: a registry written before diseases were
 recorded yields blank cells rather than breaking the page.
@@ -90,8 +90,8 @@ def latest_only(df: pd.DataFrame) -> pd.DataFrame:
     """Keep each entry's most recent submission, so nobody can shop for a lucky run.
 
     An entry is ``(name, is_baseline)``, not just the name. Sharing one namespace
-    would let somebody who submits a model called ``pca-50`` bury the published
-    ``pca-50`` baseline simply by submitting after it.
+    would let somebody who submits a model called ``Raw data`` bury the published
+    ``Raw data`` baseline simply by submitting after it.
     """
     flagged = with_submission_id(
         with_paper_link(
@@ -163,24 +163,41 @@ def _repeat_scores(row: pd.Series) -> tuple[float, ...]:
 
 
 def _entries(df: pd.DataFrame, by_id: dict[str, dict], board: Board) -> list[dict]:
-    """Per-category means for every model that covered the whole board."""
+    """Per-category means for every model covering the board's scored tasks.
+
+    A task with no finite score from any latest submission is not a leaderboard
+    task yet. Keeping it out of the coverage check lets the rest of the board be
+    ranked while still leaving the task visible in the per-task table.
+    """
     scoped = _board_registry(by_id, board)
+    latest = latest_only(df)
+    scored_tasks = {
+        _norm_id(row["task_id"])
+        for _, row in latest.iterrows()
+        if _norm_id(row["task_id"]) in board.task_ids
+        and math.isfinite(float(row["score"]))
+    }
+    if not scored_tasks:
+        return []
+
     entries = []
-    for (model, is_baseline, submitted, submission_id), rows in latest_only(df).groupby(
+    for (model, is_baseline, submitted, submission_id), rows in latest.groupby(
         [MODEL_NAME, IS_BASELINE, "submitted_at", SUBMISSION_ID]
     ):
-        if not board.task_ids.issubset({_norm_id(t) for t in rows["task_id"]}):
+        task_rows = rows[
+            rows["task_id"].map(_norm_id).isin(list(scored_tasks))
+            & rows["score"].map(lambda score: math.isfinite(float(score)))
+        ]
+        if len(task_rows["task_id"].map(_norm_id).unique()) != len(scored_tasks):
             continue
-        categories = category_means(scores_from_rows(rows, scoped))
+        categories = category_means(scores_from_rows(task_rows, scoped))
         task_scores = {
             _norm_id(row["task_id"]): float(row["score"])
-            for _, row in rows.iterrows()
-            if _norm_id(row["task_id"]) in board.task_ids
+            for _, row in task_rows.iterrows()
         }
         game_scores = {
             f"{_norm_id(row['task_id'])}:{repeat}": value
-            for _, row in rows.iterrows()
-            if _norm_id(row["task_id"]) in board.task_ids
+            for _, row in task_rows.iterrows()
             for repeat, value in enumerate(_repeat_scores(row))
         }
         entries.append(

pages/about.md

@@ -27,9 +27,12 @@ representation is any use in the clinic, and that is the gap PRIMO tries to fill
    Spearman never share a category column.
 
 The probe is identical for everyone, which is what makes the numbers comparable:
-what changes between two rows is the representation behind them. Perturbation
-residual Spearman is normalized to `[0, 1]`, with 0.5 as the uninformative
-mean-response anchor.
+what changes between two rows is the representation behind them. The rows marked
+`(baseline)` are our own submissions, scored by that same probe on the log-CPM
+expression itself, whole or cut to its most variable genes. That cut never reads
+a label, but it is made over every sample, so read those rows as a floor set by
+the raw features rather than as a competing model. Perturbation residual Spearman
+is normalized to `[0, 1]`, with 0.5 as the uninformative mean-response anchor.
 
 A board covering more than one family also shows a **Mean** of the family
 columns. It is there to give the table an order, but it does average AUROC,

pages/submit.md

@@ -32,6 +32,6 @@ scored: you get ranked on every **board** whose tasks you covered in full, and
 your numbers still show up in each board's **per-task** table, so nothing you
 send is thrown away.
 
-**Your first target is the baselines.** We run our own reference submissions,
-linear probes on the most variable genes, and they sit on the boards labelled
-`(baseline)`. Beating them is the bar to clear.
+**Your first target is the baselines.** We run our own reference submissions on
+the log-CPM expression itself, whole or cut down to its most variable genes, and
+they sit on the boards labelled `(baseline)`. Beating them is the bar to clear.

render.py

@@ -129,8 +129,8 @@ def _leading(top) -> str:
     """The card's mini-ranking, with the same empty/baseline states as before.
 
     An empty board says "be the first"; a board held only by our baselines says
-    "beat the baseline" instead, because "be the first" misleads once a PCA
-    already holds a score.
+    "beat the baseline" instead, because "be the first" misleads once the raw
+    expression already holds a score.
     """
     parts = ['<p class="pm-over" style="margin-top:14px">Leading (Elo)</p>']
     if not top:
@@ -353,7 +353,7 @@ def render_board(board: Board | None, df: pd.DataFrame, by_id: dict[str, dict])
     ranked = _df_to_table(
         ranked_table(df, by_id, board),
         bold_axis=0,
-        empty="No model has covered every task of this board yet. Be the first to submit.",
+        empty="No model has covered every scored task of this board yet. Be the first to submit.",
     )
     per_task = _df_to_table(
         per_task_table(df, by_id, board),

results.py

@@ -1,15 +1,14 @@
 """Persisted leaderboard rows: the results-CSV schema and its Hugging Face IO.
 
 Kept out of ``app.py`` so the schema has one owner and so the baseline publisher
-(``benchmark/public_benchmark/baselines.py``) can append rows without importing
-Gradio. ``huggingface_hub`` is imported lazily, so the unit tests touch no
-network.
+can append rows without importing Gradio. ``huggingface_hub`` is imported lazily,
+so the unit tests touch no network.
 
-``is_baseline`` marks a reference submission we produced ourselves (a random
-embedding, a PCA of log-CPM) rather than a model somebody sent us. Baselines are
-ranked in place, never pinned: the point of showing them is that a foundation
-model can lose to a PCA, and a row pushed to the bottom of the table would hide
-exactly that.
+``is_baseline`` marks a reference submission we produced ourselves (log-CPM
+expression, whole or cut to its most variable genes) rather than a model somebody
+sent us. Baselines are ranked in place, never pinned: the point of showing them is
+that a foundation model can lose to the raw features, and a row pushed to the
+bottom of the table would hide exactly that.
 
 ``hf_username`` privately records who submitted a name, because the board keeps
 each name's LATEST rows: without an owner, anyone could supersede another team's
@@ -129,7 +128,7 @@ def owner_of(df: pd.DataFrame, model: str) -> str:
     """Who claimed this submitted model name, or ``""`` if it is free.
 
     Baselines are ignored: they live in their own namespace (``is_baseline``), so
-    publishing ``pca-50`` never stops somebody submitting a model of that name.
+    publishing ``Raw data`` never stops somebody submitting a model of that name.
     """
     if df.empty:
         return ""

test_leaderboard.py

@@ -33,6 +33,15 @@ def test_full_board_coverage_is_ranked_and_partial_is_not(registry, named, resul
     assert list(board["Model"]) == ["full"]
 
 
+def test_tasks_with_no_finite_score_do_not_block_rankings(registry, named, results):
+    df = results(
+        *_full("better", "2026-01-01", scores=(0.9, 0.9, 0.9, 0.9, float("nan"))),
+        *_full("worse", "2026-01-01", scores=(0.2, 0.2, 0.2, 0.2, float("nan"))),
+    )
+    board = ranked_table(df, registry, named("bulk RNA"))
+    assert list(board["Model"]) == ["better", "worse"]
+
+
 def test_a_partial_model_is_ranked_on_the_board_it_fully_covered(
     registry, named, results
 ):
@@ -119,21 +128,20 @@ def test_nullable_credits_render_as_missing(registry, named, results):
     assert board.loc[0, "Model"].is_author_submission is False
 
 
-def test_a_model_scoring_nothing_finite_ranks_below_a_bad_one(registry, named, results):
-    """NaN everywhere is "no score", which must not outrank a negative Pearson."""
+def test_a_model_scoring_nothing_finite_is_not_ranked(registry, named, results):
+    """A submission with no finite score has no scored task coverage."""
     df = results(
         *_full("nan-everywhere", "2026-01-01", scores=(float("nan"),) * 5),
         *_full("bad", "2026-01-01", scores=(-0.2,) * 5),
     )
     board = ranked_table(df, registry, named("bulk RNA"))
-    assert list(board["Model"]) == ["bad", "nan-everywhere"]
-    assert board.loc[1, "Elo"] == 1000
+    assert list(board["Model"]) == ["bad"]
 
 
-def test_a_card_shows_a_dash_rather_than_minus_infinity(registry, named, results):
+def test_a_card_is_empty_when_no_model_has_a_finite_score(registry, named, results):
     df = results(*_full("nan-everywhere", "2026-01-01", scores=(float("nan"),) * 5))
     top = top_models(df, registry, named("bulk RNA"), 3)
-    assert top[0].elo is None
+    assert top == []
 
 
 def test_ranked_is_empty_when_nobody_covered_the_board(registry, named, results):
@@ -190,11 +198,11 @@ def test_top_models_are_ordered_and_limited(registry, named, results):
     assert not any(entry.is_baseline for entry in top)
 
 
-def test_top_models_ignores_models_that_did_not_cover_the_board(
-    registry, named, results
-):
+def test_top_models_uses_the_remaining_scored_tasks(registry, named, results):
     df = results(("partial", "t001", 0.99, "2026-01-01"))
-    assert top_models(df, registry, named("bulk RNA"), 3) == []
+    assert [entry.name for entry in top_models(df, registry, named("bulk RNA"), 3)] == [
+        "partial"
+    ]
 
 
 def test_a_baseline_is_ranked_in_place_and_labelled(registry, named, flagged_results):