Spaces:
Running
Running
| import pandas as pd | |
| from app.services.results_mapping import map_oeb_results_to_contents | |
| def _request(): | |
| return { | |
| "model_name": "BIOT-frozen", | |
| "organization": "braindecode", | |
| "model_url": "https://huggingface.co/braindecode/biot", | |
| "paper_url": "", | |
| "submitted_time": "2026-06-15T12:00:00Z", | |
| "benchmark_kwargs": {"hub_repo": "braindecode/biot", "model_cls": "braindecode.models.BIOT"}, | |
| } | |
| def _df(): | |
| # two datasets x one strategy x two seeds, all completed | |
| return pd.DataFrame([ | |
| {"backbone": "BIOT", "dataset": "braindecode/bcic2a", "finetuning": "frozen", | |
| "head": "linear_head", "seed": 0, "status": "completed", | |
| "test_balanced_accuracy": 0.40, "trainable_params": 3_190_000}, | |
| {"backbone": "BIOT", "dataset": "braindecode/bcic2a", "finetuning": "frozen", | |
| "head": "linear_head", "seed": 1, "status": "completed", | |
| "test_balanced_accuracy": 0.60, "trainable_params": 3_190_000}, | |
| {"backbone": "BIOT", "dataset": "braindecode/physionet", "finetuning": "frozen", | |
| "head": "linear_head", "seed": 0, "status": "completed", | |
| "test_balanced_accuracy": 0.50, "trainable_params": 3_190_000}, | |
| ]) | |
| def test_one_row_per_strategy(): | |
| rows = map_oeb_results_to_contents(_df(), _request()) | |
| assert len(rows) == 1 | |
| row = rows[0] | |
| assert row["fullname"] == "BIOT-frozen" | |
| assert row["adapter"] == "probe" # oeb "frozen" linear probing -> arena "probe" | |
| assert row["Architecture"] == "BIOT" | |
| # mean over seeds: bcic2a = (0.40+0.60)/2 = 0.50 (fraction) | |
| assert abs(row["bcic2a_accuracy"] - 0.50) < 1e-9 | |
| assert abs(row["physionet_accuracy"] - 0.50) < 1e-9 | |
| # Average over available datasets * 100 = 50.0 | |
| assert abs(row["Average ⬆️"] - 50.0) < 1e-9 | |
| assert abs(row["#Params (M)"] - 3.19) < 1e-6 | |
| assert row["Available on the hub"] is True | |
| def test_failed_rows_ignored(): | |
| df = _df() | |
| df.loc[len(df)] = {"backbone": "BIOT", "dataset": "braindecode/tuab", "finetuning": "frozen", | |
| "head": "linear_head", "seed": 0, "status": "failed", | |
| "exception": "boom", "test_balanced_accuracy": None, "trainable_params": None} | |
| rows = map_oeb_results_to_contents(df, _request()) | |
| assert rows[0].get("tuab_accuracy", 0) == 0 # failed dataset not scored | |
| def test_multiple_strategies_multiple_rows(): | |
| df = _df() | |
| extra = df.copy(); extra["finetuning"] = "lora" | |
| rows = map_oeb_results_to_contents(pd.concat([df, extra], ignore_index=True), _request()) | |
| assert {r["adapter"] for r in rows} == {"probe", "lora"} # frozen aliased to probe; lora unchanged | |
| def test_all_failed_run_has_no_metric_column(): | |
| # When every experiment fails, oeb's DataFrame has NO test_balanced_accuracy | |
| # column at all. Mapping must return [] (not KeyError), so the worker reports | |
| # the clean "no completed results" failure. (Regression: live worker test.) | |
| df = pd.DataFrame([ | |
| {"backbone": "BIOT", "dataset": "braindecode/seed_v", "finetuning": "frozen", | |
| "head": "linear_head", "seed": 0, "status": "failed", "exception": "could not load weights"}, | |
| ]) | |
| assert map_oeb_results_to_contents(df, _request()) == [] | |
| def test_metric_column_all_nan(): | |
| df = _df() | |
| df["test_balanced_accuracy"] = None | |
| assert map_oeb_results_to_contents(df, _request()) == [] | |