🐛 Change from a simple to a weighted average

src/display/text_blocks.py

@@ -49,5 +49,5 @@ Each benchmark measures the performance of the coding agent on different tasks:
 
 Higher scores indicate better performance on the benchmarks.
 If an agent scores better on a given benchmark than another, it can be generally considered to be better at those kinds of tasks.
-We take a simple average of these scores so you can quickly compare the performance of different coding agents, but this is a relative score and the average itself is meaningless on its own.
+We take a weighted average of these scores so you can quickly compare the performance of different coding agents, but this is a relative score and the average itself is meaningless on its own.
 """

src/leaderboard.py

@@ -35,7 +35,7 @@ def get_leaderboard_df():
         model_lookup[result.model.repo] = result.model
         if pair not in benchmark_lookup:
             benchmark_lookup[pair] = {}
-        benchmark_lookup[pair][result.benchmark.name] = round(result.metrics.score * 100, 1)
+        benchmark_lookup[pair][result.benchmark.name] = (round(result.metrics.score * 100, 1), result.benchmark.num_tasks)
     
     # Collect results into df rows
     rows = []
@@ -43,7 +43,7 @@ def get_leaderboard_df():
     for pair, benchmarks in benchmark_lookup.items():
         model = model_lookup[pair[0]]
         harness = harness_lookup[pair[1]]
-        avg_score = sum(benchmarks.values()) / len(benchmarks)
+        avg_score = sum([score * size for score, size in benchmarks.values()]) / sum([size for _, size in benchmarks.values()])
         row = {
             " ": "🟠" if model.is_oss and harness.is_oss else "🔶",
             "Model": f'[{model.repo}]({model.url})',
@@ -55,7 +55,7 @@ def get_leaderboard_df():
             "Avg Score": round(avg_score, 1),
         }
         for benchmark_name in sorted(benchmark_names, key=lambda x: (0 if x == "swe-bench-verified" else 1)):
-            row[benchmark_name] = benchmarks.get(benchmark_name, "")
+            row[benchmark_name] = benchmarks.get(benchmark_name, "")[0]
         rows.append(row)
     
     leaderboard_df = pd.DataFrame(rows).sort_values("Avg Score", ascending=False).fillna("")