Spaces:

lanczos
/

graphtestbed

Running

Zhu Jiajun (jz28583) Claude Opus 4.7 (1M context) commited on Apr 21

Commit

ab28b31

1 Parent(s): bf48fd7

Overall: only complete agents get an average; rest sink to bottom

Previously, an agent with one task scored had its single score shown as
the average — making 'autopipe-v2 → figraph 0.824' appear ranked above
fully-covered agents with averages in the 0.5s. That hides the real
ranking from anyone scanning the page.

New rule: average is computed only when the agent has scored on every
task in the manifest. Incomplete agents render '—' and sort to the
bottom (still ordered among themselves by # tasks covered, then name).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (1) hide show

server/api.py +30 -11

server/api.py CHANGED Viewed

@@ -411,9 +411,9 @@ def leaderboard(task: str):
 @app.get("/leaderboard")
 def leaderboard_all():
-    """Cross-task average per agent. Average is over tasks the agent has
-    submitted to (not over all tasks), so a one-task agent isn't penalized
-    by N/A on others. Sorted by average desc; ties broken by # tasks covered."""
     manifest = _manifest()
     tasks = sorted(manifest)
     conn = _db()
@@ -432,14 +432,22 @@ def leaderboard_all():
         covered = [t for t in tasks if t in scores]
         if not covered:
             continue
-        avg = sum(scores[t] for t in covered) / len(covered)
         out.append({
             "agent": agent,
-            "average": round(avg, 3),
             "n_tasks": len(covered),
             "per_task": {t: scores.get(t) for t in tasks},
         })
-    out.sort(key=lambda r: (-r["average"], -r["n_tasks"], r["agent"]))
     return jsonify({"tasks": tasks, "rows": out})
@@ -960,7 +968,9 @@ _LANDING_TMPL = r"""<!doctype html>
                   {% if v is not none %}{{ "%.3f"|format(v) }}{% else %}<span class="muted">—</span>{% endif %}
                 </td>
                 {% endfor %}
-                <td class="score">{{ "%.3f"|format(r.average) }}</td>
               </tr>
               {% endfor %}
             {% else %}
@@ -1332,21 +1342,30 @@ def landing():
         n_subs_total += sum(r["n_subs"] for r in tasks[-1]["rows"])
     conn.close()
-    # Cross-task average per agent (only over tasks they've submitted to).
     by_agent: dict[str, dict[str, float]] = {}
     for t in tasks:
         for r in t["rows"]:
             by_agent.setdefault(r["agent"], {})[t["name"]] = r["primary"]
     overall_rows = []
     for agent, scores in by_agent.items():
-        avg = sum(scores.values()) / len(scores)
         overall_rows.append({
             "agent": agent,
-            "average": round(avg, 3),
             "n_tasks": len(scores),
             "per_task": {t["name"]: scores.get(t["name"]) for t in tasks},
         })
-    overall_rows.sort(key=lambda r: (-r["average"], -r["n_tasks"], r["agent"]))
     base_url = request.url_root.rstrip("/")

 @app.get("/leaderboard")
 def leaderboard_all():
+    """Cross-task average per agent. The average is only computed for agents
+    that have a score on every task — an incomplete agent shows '—' and ranks
+    below all complete ones (ties broken by agent name for stability)."""
     manifest = _manifest()
     tasks = sorted(manifest)
     conn = _db()
         covered = [t for t in tasks if t in scores]
         if not covered:
             continue
+        complete = len(covered) == len(tasks)
+        avg = sum(scores[t] for t in covered) / len(covered) if complete else None
         out.append({
             "agent": agent,
+            "average": round(avg, 3) if avg is not None else None,
             "n_tasks": len(covered),
             "per_task": {t: scores.get(t) for t in tasks},
         })
+    # Complete agents first (sorted by average desc), then incomplete ones at
+    # the bottom (sorted by # tasks covered desc, then name).
+    out.sort(key=lambda r: (
+        0 if r["average"] is not None else 1,
+        -(r["average"] if r["average"] is not None else 0),
+        -r["n_tasks"],
+        r["agent"],
+    ))
     return jsonify({"tasks": tasks, "rows": out})
                   {% if v is not none %}{{ "%.3f"|format(v) }}{% else %}<span class="muted">—</span>{% endif %}
                 </td>
                 {% endfor %}
+                <td class="score">
+                  {% if r.average is not none %}{{ "%.3f"|format(r.average) }}{% else %}<span class="muted">—</span>{% endif %}
+                </td>
               </tr>
               {% endfor %}
             {% else %}
         n_subs_total += sum(r["n_subs"] for r in tasks[-1]["rows"])
     conn.close()
+    # Cross-task average per agent. Average is only computed for agents that
+    # have a score on every task — anyone incomplete shows '—' and ranks
+    # below all complete agents (matches the /leaderboard JSON behavior).
     by_agent: dict[str, dict[str, float]] = {}
     for t in tasks:
         for r in t["rows"]:
             by_agent.setdefault(r["agent"], {})[t["name"]] = r["primary"]
     overall_rows = []
+    n_total = len(tasks)
     for agent, scores in by_agent.items():
+        complete = len(scores) == n_total
+        avg = round(sum(scores.values()) / len(scores), 3) if complete else None
         overall_rows.append({
             "agent": agent,
+            "average": avg,
             "n_tasks": len(scores),
             "per_task": {t["name"]: scores.get(t["name"]) for t in tasks},
         })
+    overall_rows.sort(key=lambda r: (
+        0 if r["average"] is not None else 1,
+        -(r["average"] if r["average"] is not None else 0),
+        -r["n_tasks"],
+        r["agent"],
+    ))
     base_url = request.url_root.rstrip("/")