karimox commited on
Commit
1dc279c
·
verified ·
1 Parent(s): 6286da1

Nicer intro: 3-step submit flow + data download link

Browse files
Files changed (1) hide show
  1. app.py +19 -11
app.py CHANGED
@@ -41,14 +41,15 @@ DETAIL_COLUMNS = ["dataset_id", "status", "skill"]
41
 
42
  def _read_results() -> pd.DataFrame:
43
  from huggingface_hub import hf_hub_download
 
44
 
45
  try:
46
  path = hf_hub_download(
47
  RESULTS_REPO, RESULTS_FILE, repo_type="dataset", token=TOKEN
48
  )
49
- return pd.read_csv(path)
50
- except Exception:
51
  return pd.DataFrame(columns=RESULT_COLUMNS)
 
52
 
53
 
54
  def _append_results(rows: list[dict]) -> None:
@@ -125,8 +126,8 @@ def leaderboard() -> pd.DataFrame:
125
  joined from the task registry at read time; if that fetch fails the board
126
  renders empty rather than showing a wrong ranking.
127
  """
128
- df = _read_results()
129
  try:
 
130
  datasets = manifest_ids(fetch_manifest(TOKEN))
131
  registry = scoreable_tasks(fetch_tasks_registry(TOKEN), datasets)
132
  by_id = {_norm_id(task["task_id"]): task for task in registry}
@@ -258,14 +259,21 @@ def evaluate(submission_path: str, model_name: str):
258
  def build_demo() -> gr.Blocks:
259
  with gr.Blocks(title="Lodestar Benchmark") as demo:
260
  gr.Markdown(
261
- "# 🧬 Lodestar\n"
262
- "Blind benchmark for transcriptomic foundation models. Embed **every** "
263
- "dataset and upload **one** file; a fixed linear probe scores each dataset "
264
- "and rolls up into a per-facet skill leaderboard. Only full-coverage "
265
- "submissions are ranked.\n\n"
266
- "Submission = CSV / TSV / Parquet with a `dataset_id` column, a `sample_id` "
267
- "column, and one column per embedding dim; or NPZ with `dataset_ids`, "
268
- "`sample_ids`, `embeddings` arrays."
 
 
 
 
 
 
 
269
  )
270
  model_tb = gr.Textbox(label="Model name", placeholder="e.g. eva-rna-v1")
271
  file_in = gr.File(
 
41
 
42
  def _read_results() -> pd.DataFrame:
43
  from huggingface_hub import hf_hub_download
44
+ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
45
 
46
  try:
47
  path = hf_hub_download(
48
  RESULTS_REPO, RESULTS_FILE, repo_type="dataset", token=TOKEN
49
  )
50
+ except (RepositoryNotFoundError, EntryNotFoundError):
 
51
  return pd.DataFrame(columns=RESULT_COLUMNS)
52
+ return pd.read_csv(path)
53
 
54
 
55
  def _append_results(rows: list[dict]) -> None:
 
126
  joined from the task registry at read time; if that fetch fails the board
127
  renders empty rather than showing a wrong ranking.
128
  """
 
129
  try:
130
+ df = _read_results()
131
  datasets = manifest_ids(fetch_manifest(TOKEN))
132
  registry = scoreable_tasks(fetch_tasks_registry(TOKEN), datasets)
133
  by_id = {_norm_id(task["task_id"]): task for task in registry}
 
259
  def build_demo() -> gr.Blocks:
260
  with gr.Blocks(title="Lodestar Benchmark") as demo:
261
  gr.Markdown(
262
+ "# 🧬 Lodestar\n\n"
263
+ "A **blind benchmark for transcriptomic foundation models** — grade "
264
+ "your model's patient-level embeddings against real clinical signal, "
265
+ "without ever seeing the labels.\n\n"
266
+ "**Submit in 3 steps:**\n"
267
+ "1. **Get the data** download the opaque datasets from "
268
+ "[ScientaLab/lodestar](https://huggingface.co/datasets/ScientaLab/lodestar)"
269
+ " (start with its `datasets.yaml`).\n"
270
+ "2. **Embed every dataset** → build **one** file: `dataset_id`, "
271
+ "`sample_id`, then one column per embedding dim (`e0`, `e1`, …). "
272
+ "CSV / TSV / Parquet, or NPZ.\n"
273
+ "3. **Upload below**, name your model, and hit **Evaluate**.\n\n"
274
+ "A fixed linear probe scores each hidden task (AUROC or Pearson), "
275
+ "rescaled to a 0–1 skill and rolled up per specialty. **Only "
276
+ "submissions covering every dataset are ranked.**"
277
  )
278
  model_tb = gr.Textbox(label="Model name", placeholder="e.g. eva-rna-v1")
279
  file_in = gr.File(