Spaces:
Running
Running
Nicer intro: 3-step submit flow + data download link
Browse files
app.py
CHANGED
|
@@ -41,14 +41,15 @@ DETAIL_COLUMNS = ["dataset_id", "status", "skill"]
|
|
| 41 |
|
| 42 |
def _read_results() -> pd.DataFrame:
|
| 43 |
from huggingface_hub import hf_hub_download
|
|
|
|
| 44 |
|
| 45 |
try:
|
| 46 |
path = hf_hub_download(
|
| 47 |
RESULTS_REPO, RESULTS_FILE, repo_type="dataset", token=TOKEN
|
| 48 |
)
|
| 49 |
-
|
| 50 |
-
except Exception:
|
| 51 |
return pd.DataFrame(columns=RESULT_COLUMNS)
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
def _append_results(rows: list[dict]) -> None:
|
|
@@ -125,8 +126,8 @@ def leaderboard() -> pd.DataFrame:
|
|
| 125 |
joined from the task registry at read time; if that fetch fails the board
|
| 126 |
renders empty rather than showing a wrong ranking.
|
| 127 |
"""
|
| 128 |
-
df = _read_results()
|
| 129 |
try:
|
|
|
|
| 130 |
datasets = manifest_ids(fetch_manifest(TOKEN))
|
| 131 |
registry = scoreable_tasks(fetch_tasks_registry(TOKEN), datasets)
|
| 132 |
by_id = {_norm_id(task["task_id"]): task for task in registry}
|
|
@@ -258,14 +259,21 @@ def evaluate(submission_path: str, model_name: str):
|
|
| 258 |
def build_demo() -> gr.Blocks:
|
| 259 |
with gr.Blocks(title="Lodestar Benchmark") as demo:
|
| 260 |
gr.Markdown(
|
| 261 |
-
"# 🧬 Lodestar\n"
|
| 262 |
-
"
|
| 263 |
-
"
|
| 264 |
-
"
|
| 265 |
-
"
|
| 266 |
-
"
|
| 267 |
-
"
|
| 268 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
)
|
| 270 |
model_tb = gr.Textbox(label="Model name", placeholder="e.g. eva-rna-v1")
|
| 271 |
file_in = gr.File(
|
|
|
|
| 41 |
|
| 42 |
def _read_results() -> pd.DataFrame:
|
| 43 |
from huggingface_hub import hf_hub_download
|
| 44 |
+
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 45 |
|
| 46 |
try:
|
| 47 |
path = hf_hub_download(
|
| 48 |
RESULTS_REPO, RESULTS_FILE, repo_type="dataset", token=TOKEN
|
| 49 |
)
|
| 50 |
+
except (RepositoryNotFoundError, EntryNotFoundError):
|
|
|
|
| 51 |
return pd.DataFrame(columns=RESULT_COLUMNS)
|
| 52 |
+
return pd.read_csv(path)
|
| 53 |
|
| 54 |
|
| 55 |
def _append_results(rows: list[dict]) -> None:
|
|
|
|
| 126 |
joined from the task registry at read time; if that fetch fails the board
|
| 127 |
renders empty rather than showing a wrong ranking.
|
| 128 |
"""
|
|
|
|
| 129 |
try:
|
| 130 |
+
df = _read_results()
|
| 131 |
datasets = manifest_ids(fetch_manifest(TOKEN))
|
| 132 |
registry = scoreable_tasks(fetch_tasks_registry(TOKEN), datasets)
|
| 133 |
by_id = {_norm_id(task["task_id"]): task for task in registry}
|
|
|
|
| 259 |
def build_demo() -> gr.Blocks:
|
| 260 |
with gr.Blocks(title="Lodestar Benchmark") as demo:
|
| 261 |
gr.Markdown(
|
| 262 |
+
"# 🧬 Lodestar\n\n"
|
| 263 |
+
"A **blind benchmark for transcriptomic foundation models** — grade "
|
| 264 |
+
"your model's patient-level embeddings against real clinical signal, "
|
| 265 |
+
"without ever seeing the labels.\n\n"
|
| 266 |
+
"**Submit in 3 steps:**\n"
|
| 267 |
+
"1. **Get the data** → download the opaque datasets from "
|
| 268 |
+
"[ScientaLab/lodestar](https://huggingface.co/datasets/ScientaLab/lodestar)"
|
| 269 |
+
" (start with its `datasets.yaml`).\n"
|
| 270 |
+
"2. **Embed every dataset** → build **one** file: `dataset_id`, "
|
| 271 |
+
"`sample_id`, then one column per embedding dim (`e0`, `e1`, …). "
|
| 272 |
+
"CSV / TSV / Parquet, or NPZ.\n"
|
| 273 |
+
"3. **Upload below**, name your model, and hit **Evaluate**.\n\n"
|
| 274 |
+
"A fixed linear probe scores each hidden task (AUROC or Pearson), "
|
| 275 |
+
"rescaled to a 0–1 skill and rolled up per specialty. **Only "
|
| 276 |
+
"submissions covering every dataset are ranked.**"
|
| 277 |
)
|
| 278 |
model_tb = gr.Textbox(label="Model name", placeholder="e.g. eva-rna-v1")
|
| 279 |
file_in = gr.File(
|