bruAristimunha commited on
Commit
5d5fb3c
·
1 Parent(s): 19eb9db

change: paper results win on conflict (official results immutable)

Browse files

Per request, flip the leaderboard merge so the bundled paper results are
authoritative: on a (model, adapter) clash the paper row wins, and community
submissions from braindecode/contents can only ADD new model/strategy combos,
never override an official result.

backend/app/services/leaderboard.py CHANGED
@@ -30,13 +30,14 @@ def _merge_results(
30
  ) -> List[Dict[str, Any]]:
31
  """Merge curated/base results with new ones, deduped by (model, adapter).
32
 
33
- ``base`` is the bundled real/paper results; ``overlay`` is new community
34
- results from the HF contents dataset. Overlay rows override base rows that
35
- share the same (fullname, adapter) so a freshly evaluated result supersedes
36
- the baseline, while every other base result is preserved.
 
37
  """
38
  merged: Dict[tuple, Dict[str, Any]] = {}
39
- for row in list(base) + list(overlay): # overlay last => wins on conflict
40
  merged[(row.get("fullname"), row.get("adapter"))] = row
41
  return list(merged.values())
42
 
@@ -48,9 +49,10 @@ class LeaderboardService:
48
  async def fetch_raw_data(self) -> List[Dict[str, Any]]:
49
  """Merge the bundled real/paper results with new community results.
50
 
51
- ``sample_results.json`` (the paper results) is the immutable base. New
52
- results from the HF ``contents`` dataset are overlaid on top so the board
53
- shows BOTH; on a (model, adapter) clash the new result wins. Falls back
 
54
  to base-only when the contents dataset isn't available yet.
55
  """
56
  try:
 
30
  ) -> List[Dict[str, Any]]:
31
  """Merge curated/base results with new ones, deduped by (model, adapter).
32
 
33
+ ``base`` is the bundled real/paper results (authoritative & immutable);
34
+ ``overlay`` is new community results from the HF contents dataset. The paper
35
+ results always win on a (fullname, adapter) clash, so community submissions
36
+ can only ADD new model/strategy combinations they never override an
37
+ official result.
38
  """
39
  merged: Dict[tuple, Dict[str, Any]] = {}
40
+ for row in list(overlay) + list(base): # base last => paper results win on conflict
41
  merged[(row.get("fullname"), row.get("adapter"))] = row
42
  return list(merged.values())
43
 
 
49
  async def fetch_raw_data(self) -> List[Dict[str, Any]]:
50
  """Merge the bundled real/paper results with new community results.
51
 
52
+ ``sample_results.json`` (the paper results) is the authoritative, immutable
53
+ base. New results from the HF ``contents`` dataset are overlaid so the
54
+ board shows BOTH; on a (model, adapter) clash the PAPER result wins, so
55
+ community submissions can only add new model/strategy combos. Falls back
56
  to base-only when the contents dataset isn't available yet.
57
  """
58
  try:
backend/tests/test_leaderboard_merge.py CHANGED
@@ -1,21 +1,21 @@
1
  from app.services.leaderboard import _merge_results
2
 
3
 
4
- def test_merge_overlay_wins_on_conflict():
5
  base = [
6
  {"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 30.0},
7
  {"fullname": "LaBraM", "adapter": "probe", "Average ⬆️": 12.0},
8
  ]
9
  overlay = [
10
- {"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 40.0}, # overrides base
11
- {"fullname": "MyModel", "adapter": "lora", "Average ⬆️": 50.0}, # brand new
12
  ]
13
  out = _merge_results(base, overlay)
14
  by = {(r["fullname"], r["adapter"]): r for r in out}
15
  assert len(out) == 3
16
- assert by[("BIOT", "full_finetune")]["Average ⬆️"] == 40.0 # new result wins
17
  assert by[("LaBraM", "probe")]["Average ⬆️"] == 12.0 # paper result preserved
18
- assert ("MyModel", "lora") in by # new model added
19
 
20
 
21
  def test_merge_empty_overlay_returns_base():
 
1
  from app.services.leaderboard import _merge_results
2
 
3
 
4
+ def test_merge_paper_base_wins_on_conflict():
5
  base = [
6
  {"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 30.0},
7
  {"fullname": "LaBraM", "adapter": "probe", "Average ⬆️": 12.0},
8
  ]
9
  overlay = [
10
+ {"fullname": "BIOT", "adapter": "full_finetune", "Average ⬆️": 40.0}, # same combo as a paper row
11
+ {"fullname": "MyModel", "adapter": "lora", "Average ⬆️": 50.0}, # brand new combo
12
  ]
13
  out = _merge_results(base, overlay)
14
  by = {(r["fullname"], r["adapter"]): r for r in out}
15
  assert len(out) == 3
16
+ assert by[("BIOT", "full_finetune")]["Average ⬆️"] == 30.0 # paper result wins (immutable)
17
  assert by[("LaBraM", "probe")]["Average ⬆️"] == 12.0 # paper result preserved
18
+ assert ("MyModel", "lora") in by # new combo added
19
 
20
 
21
  def test_merge_empty_overlay_returns_base():