✨ Add new/updated metrics

app.py

@@ -61,7 +61,8 @@ def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     
-    # Make ColumnFilter choices from md format
+    # Make ColumnFilter choices
+    label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
     dataset_choices = sorted({(extract_body(v), v) for v in dataframe["Dataset"]})
     
     return Leaderboard(
@@ -73,6 +74,7 @@ def init_leaderboard(dataframe):
         datatype="markdown",
         search_columns=SEARCH_COLUMNS,
         filter_columns=[
+            ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
             ColumnFilter(label="Dataset", column="Dataset", type="checkboxgroup", choices=dataset_choices),
             ColumnFilter(label="Model License", column="Model License", type="checkboxgroup"),
             ColumnFilter(label="Harness License", column="Harness License", type="checkboxgroup"),

results/claude-opus-4-7-internal.json

@@ -24,8 +24,6 @@
         "url": "https://www.anthropic.com/news/claude-opus-4-7"
     },
     "metrics": {
-        "score": 0.876,
-        "time": null,
-        "costUSD": null
+        "score": 0.876
     }
 }

results/qwen3-6-35b-internal.json

@@ -24,8 +24,6 @@
         "url": "https://qwen.ai/blog?id=qwen3.6-35b-a3b"
     },
     "metrics": {
-        "score": 0.734,
-        "time": null,
-        "costUSD": null
+        "score": 0.734
     }
 }

results/qwen3-6-35b-nvfp4-claude-code.json

@@ -37,8 +37,20 @@
         "url": "https://github.com/harbor-framework/harbor"
     },
     "metrics": {
-        "score": 0.632,
-        "time": 21600,
-        "costUSD": 48.00
+        "n_tasks": 500,
+        "n_errors": 1,
+        "score": 0.63,
+        "n_input_tokens": 1106618897,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 5733245,
+        "n_total_tokens": 1112352142,
+        "time_seconds": 122808,
+        "cost_usd": 34.11,
+        "mean_input_tokens_per_task": 2213237,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 11466,
+        "mean_tokens_per_task": 2224704,
+        "mean_cost_usd_per_task": 0.07,
+        "mean_time_seconds_per_task": 245
     }
 }

results/qwen3-6-35b-nvfp4-opencode.json

@@ -38,8 +38,20 @@
         "url": "https://github.com/harbor-framework/harbor"
     },
     "metrics": {
-        "score": 0.548,
-        "time": 29940,
-        "costUSD": 66.53
+        "n_tasks": 500,
+        "n_errors": 4,
+        "score": 0.55,
+        "n_input_tokens": 469806650,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4937761,
+        "n_total_tokens": 474744411,
+        "time_seconds": 120473,
+        "cost_usd": 29.75,
+        "mean_input_tokens_per_task": 939613,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 9875,
+        "mean_tokens_per_task": 949488,
+        "mean_cost_usd_per_task": 0.06,
+        "mean_time_seconds_per_task": 240
     }
 }

results/qwen3-6-36b-nvfp4-pi.json

@@ -38,8 +38,20 @@
         "url": "https://github.com/harbor-framework/harbor"
     },
     "metrics": {
-        "score": 0.650,
-        "time": 23160,
-        "costUSD": 51.47
+        "n_tasks": 500,
+        "n_errors": 6,
+        "score": 0.65,
+        "n_input_tokens": 791183735,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 6333798,
+        "n_total_tokens": 797517533,
+        "time_seconds": 154531,
+        "cost_usd": 38.16,
+        "mean_input_tokens_per_task": 1582367,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 12667,
+        "mean_tokens_per_task": 1595035,
+        "mean_cost_usd_per_task": 0.08,
+        "mean_time_seconds_per_task": 309
     }
 }

src/leaderboard.py

@@ -7,6 +7,7 @@ from src.models import Result
 RESULTS_DIR = Path(__file__).parent.parent / "results"
 
 DISPLAY_BY_DEFAULT = [
+    " ",
     "Dataset",
     "Harness",
     "Model",
@@ -43,6 +44,7 @@ def get_leaderboard_df():
     for result in results:
         rows.append(
             {
+                " ": "🟠" if result.model.is_oss and result.harness.is_oss else "🔶",
                 "Dataset": f'[{result.dataset.name}]({result.dataset.url})',
                 "Harness": f'[{result.harness.name}]({result.harness.url})<sup>*</sup>' if result.harness.name == "internal" else f'[{result.harness.name}]({result.harness.url})',
                 "Model": result.model.name,
@@ -51,8 +53,10 @@ def get_leaderboard_df():
                 "Skills": str(result.harness.skills) if result.harness.skills else "None",
                 "Environment": f'[{result.environment.name}]({result.environment.url})<sup>*</sup>' if result.environment.name == "internal" else f'[{result.environment.name}]({result.environment.url})',
                 "Score": result.metrics.score,
-                "Cost (USD)": result.metrics.costUSD,
-                "Time": format_time(result.metrics.time),
+                "Avg Cost Per Task (USD)": result.metrics.mean_cost_usd_per_task,
+                "Avg Seconds Per Task": result.metrics.mean_time_seconds_per_task,
+                "Avg Input Tokens Per Task": result.metrics.mean_input_tokens_per_task,
+                "Avg Output Tokens Per Task": result.metrics.mean_output_tokens_per_task,
                 "Model License": "FOSS" if result.model.is_oss else "Proprietary",
                 "Harness License": "FOSS" if result.harness.is_oss else "Proprietary",
                 "Model Num Params (B)": result.model.num_params,

src/models.py

@@ -33,9 +33,22 @@ class Environment(BaseModel):
 
 
 class Metrics(BaseModel):
+    
     score: float
-    time: Optional[int] = None
-    costUSD: Optional[float] = None
+    n_tasks: Optional[int] = None
+    n_errors: Optional[int] = None
+    n_input_tokens: Optional[int] = None
+    n_cache_tokens: Optional[int] = None
+    n_output_tokens: Optional[int] = None
+    n_total_tokens: Optional[int] = None
+    time_seconds: Optional[int] = None
+    cost_usd: Optional[float] = None
+    mean_input_tokens_per_task: Optional[int] = None
+    mean_cache_tokens_per_task: Optional[int] = None
+    mean_output_tokens_per_task: Optional[int] = None
+    mean_tokens_per_task: Optional[int] = None
+    mean_cost_usd_per_task: Optional[float] = None
+    mean_time_seconds_per_task: Optional[int] = None
 
 
 class Result(BaseModel):