Add RobloxQA_OpenEnded

src/about.py

@@ -8,4 +8,6 @@ Tracking LLM capabilities regarding Roblox game development.
 Benchmarks:
 
 - [RobloxQA](https://huggingface.co/datasets/boatbomber/RobloxQA-v1.0): Multiple choice question answering about Roblox APIs and concepts.
+- [RobloxQA_OpenEnded](https://huggingface.co/datasets/boatbomber/RobloxQA-v1.0): Question answering about Roblox APIs and concepts without giving the multiple choices. \
+Correctness judged by an LLM by comparing the generated answer to the correct answer choice.
 """

src/leaderboard/populate.py

@@ -42,7 +42,7 @@ def load_results() -> pd.DataFrame:
 
             for c in COLUMNS:
                 if c.name not in evaluation:
-                    evaluation[c.name] = None
+                    evaluation[c.name] = c.default
 
             data.append(evaluation)
 

src/leaderboard/utils.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Any
 
 
 @dataclass
@@ -9,6 +10,7 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
     searchable: bool = False
+    default: Any = None
 
 
 ## Leaderboard columns
@@ -16,6 +18,7 @@ COLUMNS = [
     ColumnContent("Model", type="str", displayed_by_default=True, never_hidden=True, searchable=True),
     ColumnContent("Precision", type="str", displayed_by_default=False),
     ColumnContent("Params (B)", type="number", displayed_by_default=True),
-    ColumnContent("Average", type="number", displayed_by_default=True),
-    ColumnContent("RobloxQA", type="number", displayed_by_default=True),
+    ColumnContent("Average", type="number", displayed_by_default=True, default=0),
+    ColumnContent("RobloxQA", type="number", displayed_by_default=True, default=0),
+    ColumnContent("RobloxQA_OpenEnded", type="number", displayed_by_default=True, default=0),
 ]