fewshot_as_multiturn

src/leaderboard/read_evals.py

@@ -47,6 +47,7 @@ class EvalResult:
         n_shot = data.get("n-shot")
         start_date = data.get("date", 0)
         chat_template = data.get("chat_template", None)
+        fewshot_as_multiturn = data.get("fewshot_as_multiturn", True)
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
@@ -78,6 +79,8 @@ class EvalResult:
 
         if chat_template:
             org_and_model += ",chat"
+            if not fewshot_as_multiturn:
+                org_and_model += ",no-multiturn"
 
         org_and_model = org_and_model.split("/", 1)