Spaces:
Running
Running
Commit ·
5d5fb3c
1
Parent(s): 19eb9db
change: paper results win on conflict (official results immutable)
Browse filesPer request, flip the leaderboard merge so the bundled paper results are
authoritative: on a (model, adapter) clash the paper row wins, and community
submissions from braindecode/contents can only ADD new model/strategy combos,
never override an official result.
backend/app/services/leaderboard.py
CHANGED
|
@@ -30,13 +30,14 @@ def _merge_results(
|
|
| 30 |
) -> List[Dict[str, Any]]:
|
| 31 |
"""Merge curated/base results with new ones, deduped by (model, adapter).
|
| 32 |
|
| 33 |
-
``base`` is the bundled real/paper results
|
| 34 |
-
results from the HF contents dataset.
|
| 35 |
-
|
| 36 |
-
|
|
|
|
| 37 |
"""
|
| 38 |
merged: Dict[tuple, Dict[str, Any]] = {}
|
| 39 |
-
for row in list(
|
| 40 |
merged[(row.get("fullname"), row.get("adapter"))] = row
|
| 41 |
return list(merged.values())
|
| 42 |
|
|
@@ -48,9 +49,10 @@ class LeaderboardService:
|
|
| 48 |
async def fetch_raw_data(self) -> List[Dict[str, Any]]:
|
| 49 |
"""Merge the bundled real/paper results with new community results.
|
| 50 |
|
| 51 |
-
``sample_results.json`` (the paper results) is the
|
| 52 |
-
results from the HF ``contents`` dataset are overlaid
|
| 53 |
-
shows BOTH; on a (model, adapter) clash the
|
|
|
|
| 54 |
to base-only when the contents dataset isn't available yet.
|
| 55 |
"""
|
| 56 |
try:
|
|
|
|
| 30 |
) -> List[Dict[str, Any]]:
|
| 31 |
"""Merge curated/base results with new ones, deduped by (model, adapter).
|
| 32 |
|
| 33 |
+
``base`` is the bundled real/paper results (authoritative & immutable);
|
| 34 |
+
``overlay`` is new community results from the HF contents dataset. The paper
|
| 35 |
+
results always win on a (fullname, adapter) clash, so community submissions
|
| 36 |
+
can only ADD new model/strategy combinations — they never override an
|
| 37 |
+
official result.
|
| 38 |
"""
|
| 39 |
merged: Dict[tuple, Dict[str, Any]] = {}
|
| 40 |
+
for row in list(overlay) + list(base): # base last => paper results win on conflict
|
| 41 |
merged[(row.get("fullname"), row.get("adapter"))] = row
|
| 42 |
return list(merged.values())
|
| 43 |
|
|
|
|
| 49 |
async def fetch_raw_data(self) -> List[Dict[str, Any]]:
|
| 50 |
"""Merge the bundled real/paper results with new community results.
|
| 51 |
|
| 52 |
+
``sample_results.json`` (the paper results) is the authoritative, immutable
|
| 53 |
+
base. New results from the HF ``contents`` dataset are overlaid so the
|
| 54 |
+
board shows BOTH; on a (model, adapter) clash the PAPER result wins, so
|
| 55 |
+
community submissions can only add new model/strategy combos. Falls back
|
| 56 |
to base-only when the contents dataset isn't available yet.
|
| 57 |
"""
|
| 58 |
try:
|
backend/tests/test_leaderboard_merge.py
CHANGED
|
@@ -1,21 +1,21 @@
|
|
| 1 |
from app.services.leaderboard import _merge_results
|
| 2 |
|
| 3 |
|
| 4 |
-
def
|
| 5 |
base = [
|
| 6 |
{"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 30.0},
|
| 7 |
{"fullname": "LaBraM", "adapter": "probe", "Average ⬆️": 12.0},
|
| 8 |
]
|
| 9 |
overlay = [
|
| 10 |
-
{"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 40.0}, #
|
| 11 |
-
{"fullname": "MyModel", "adapter": "lora", "Average ⬆️": 50.0}, # brand new
|
| 12 |
]
|
| 13 |
out = _merge_results(base, overlay)
|
| 14 |
by = {(r["fullname"], r["adapter"]): r for r in out}
|
| 15 |
assert len(out) == 3
|
| 16 |
-
assert by[("BIOT", "full_finetune")]["Average ⬆️"] ==
|
| 17 |
assert by[("LaBraM", "probe")]["Average ⬆️"] == 12.0 # paper result preserved
|
| 18 |
-
assert ("MyModel", "lora") in by # new
|
| 19 |
|
| 20 |
|
| 21 |
def test_merge_empty_overlay_returns_base():
|
|
|
|
| 1 |
from app.services.leaderboard import _merge_results
|
| 2 |
|
| 3 |
|
| 4 |
+
def test_merge_paper_base_wins_on_conflict():
|
| 5 |
base = [
|
| 6 |
{"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 30.0},
|
| 7 |
{"fullname": "LaBraM", "adapter": "probe", "Average ⬆️": 12.0},
|
| 8 |
]
|
| 9 |
overlay = [
|
| 10 |
+
{"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 40.0}, # same combo as a paper row
|
| 11 |
+
{"fullname": "MyModel", "adapter": "lora", "Average ⬆️": 50.0}, # brand new combo
|
| 12 |
]
|
| 13 |
out = _merge_results(base, overlay)
|
| 14 |
by = {(r["fullname"], r["adapter"]): r for r in out}
|
| 15 |
assert len(out) == 3
|
| 16 |
+
assert by[("BIOT", "full_finetune")]["Average ⬆️"] == 30.0 # paper result wins (immutable)
|
| 17 |
assert by[("LaBraM", "probe")]["Average ⬆️"] == 12.0 # paper result preserved
|
| 18 |
+
assert ("MyModel", "lora") in by # new combo added
|
| 19 |
|
| 20 |
|
| 21 |
def test_merge_empty_overlay_returns_base():
|