!ref suite

tasks.py

@@ -65,6 +65,7 @@ class Task:
     few_shot: int = 0
     few_shot_from: Optional[str] = None
     # results: dict[str, Any] = field(default_factory=dict)
+    outputs: Optional[list] = field(default_factory=list)
 
     def __post_init__(self):
         names = (
@@ -142,31 +143,21 @@ class Task:
         )
         return metric
 
+    @cached_property
+    def result(self) -> dict:
+        assert self.outputs, "Please run the task first."
+        return self.metric._compute(
+            responses=self.outputs, references=self.dataset[self.label_column]
+        )
+
     # @cache
     def run(
         self,
         pipeline,
     ):
-        if (outputs := pipeline(self.samples)) is None:
-            logging.warning("pipeline returns None")
-            return
-        self.outputs = outputs
-        try:
-            try:
-                result = self.metric._compute(
-                    responses=outputs, references=self.dataset[self.label_column]
-                )
-            except Exception as e:
-                result = self.metric.compute(
-                    responses=outputs, references=self.dataset[self.label_column]
-                )
-        except Exception as e:
-            result = outputs
-        # if log:
-        #     name = name or pipeline.__name__
-        #     self.results[name] = result
+        self.outputs = self.outputs or pipeline(self.samples)
 
-        return result
+        return self.result
 
 
 def multichoice(responses: Any, references: list[str]):

tlem.py

@@ -12,6 +12,7 @@ import datasets
 import pandas as pd
 from .tasks import *
 from .utils import *
+from itertools import chain
 
 
 class ReasoningMetric(evaluate.Metric):
@@ -70,33 +71,35 @@ class ReasoningMetric(evaluate.Metric):
 class Suite(EvaluationSuite):
     task_class = Task
 
+    def __getitem__(self, key) -> Task:
+        match key:
+            case str():
+                return self.suite[key]
+            # case _:
+            #     return list(chain(*self.suite.values()))[key]
+
     def run(
         self,
         model_or_pipeline: Any,
+        suite=None,
     ) -> dict[str, float]:
         self.assert_suite_nonempty()
+        if suite is None:
+            suite = self.suite
+
+        self.suite: dict[str, list[Task]]
+        results = defaultdict(dict)
+        for category, tasks in (bar := tqdm(self.suite.items())):
+            bar.desc = f"complete {category}."
+            if isinstance(tasks, dict):
+                results[category] = self.run(model_or_pipeline, tasks)
+            else:
+                for task in tasks:
+                    results[category].update(task.run(model_or_pipeline))
+                results[category] = np.mean(list(results[category].values()))
+        return results
 
-        def run_tasks(tasks):
-            for task in (bar := tqdm(tasks, leave=False)):
-                bar.desc = f"complete {task.name}."
-                if task.name not in self.cached_result:
-                    self.cached_result[task.name] = task.run(model_or_pipeline)
-            results = [self.cached_result[task.name] for task in tasks]
-            return pd.DataFrame(results).mean().to_dict()
-
-        if isinstance(self.suite, dict):
-            for category, tasks in (bar := tqdm(self.suite.items())):
-                bar.desc = f"complete {category}."
-                logging.warning(f"Combined results {category}: {run_tasks(tasks)}")
-        else:
-            logging.warning(f"Combined results: {run_tasks(self.suite)}")
-
-        return self.cached_result
-
-    def add(self, name):
-        self.load(name)
-
-    def load(self, name):
+    def get_suite(self, name) -> dict[str, Task]:
         chat = False
         match name:
             case _ if "chat" in name:
@@ -106,6 +109,8 @@ class Suite(EvaluationSuite):
                 suite = MMLU.suite(chat=chat)
             case _ if name.startswith("cmmlu"):
                 suite = CMMLU.suite(chat=chat)
+            case _ if name.startswith("ceval"):
+                suite = CEVAL.suite(chat=chat)
             case "gsm8k":
                 suite = Task(
                     dataset_name=("gsm8k", "main"),
@@ -123,8 +128,7 @@ class Suite(EvaluationSuite):
                 suite = DROP.suite()
             case "winogrande":
                 suite = Winogrande.suite()
-            case _ if name.startswith("ceval"):
-                suite = CEVAL.suite(chat=chat)
+
             case "mt_bench":
                 suite = Task(
                     dataset_name="SUSTech/mt_bench_judge",
@@ -135,16 +139,39 @@ class Suite(EvaluationSuite):
             case "MATH" | "competition_math":
                 suite = Task(
                     dataset_name="hendrycks/competition_math",
-                    split="test",
-                    prompt="This is a math problem, please think step by step and slove it: {input_column}, simplify your final answer as much as possible and surround them with $ in TeX form",
+                    prompt="This is a math problem, please think step by step and slove it: {input_column}. Simplify your final answer as much as possible and surround them with '$' in TeX form",
                     metric_name=("sustech/tlem", "MATH"),
                     input_column="problem",
                     label_column="solution",
                 )
+        if isinstance(suite, Task):
+            suite = [suite]
+        if isinstance(suite, list):
+            suite = {name: suite}
+
+        return suite
 
-        self.suite = [suite] if isinstance(suite, Task) else suite
+    def singleton(self, task):
+        try:
+            return self.tasks[self.tasks.index(task)]
+
+        except Exception as e:
+            self.tasks.append(task)
+            return self.tasks[-1]
+
+    def drop_duplicates(self, suite):
+        for category, tasks in suite.items():
+            if isinstance(tasks, dict):
+                suite[category] = self.drop_duplicates(tasks)
+            else:
+                suite[category] = [self.singleton(task) for task in tasks]
+        return suite
+
+    def load(self, name):
+        self.suite.update(self.get_suite(name))
+        self.suite = self.drop_duplicates(self.suite)
 
     def __init__(self, name="tlem"):
         super().__init__(name)
-        self.cached_result = {}
-        self.suite = []
+        self.tasks = []
+        self.suite = {}

utils.py

@@ -138,13 +138,13 @@ def extract_numeric(string, pattern=NUMERIC_IN_EN) -> str:
 def remove_boxed(s):
     if "\\boxed " in s:
         left = "\\boxed "
-        assert s[: len(left)] == left
+        assert s[: len(left)] == left, s
         return s[len(left) :]
 
     left = "\\boxed{"
 
-    assert s[: len(left)] == left
-    assert s[-1] == "}"
+    assert s[: len(left)] == left, s
+    assert s[-1] == "}", s
 
     return s[len(left) : -1]