upd

tasks.py

@@ -94,6 +94,10 @@ class Task:
     def samples(self):
         return self.dataset[self.input_column]
 
+    @cached_property
+    def labels(self):
+        return self.dataset[self.label_column]
+
     @cached_property
     def dataset(self):
         ds = (
@@ -149,12 +153,10 @@ class Task:
         )
         return metric._compute
 
-    @cached_property
+    @property
     def result(self) -> dict:
         assert self.outputs, "Please run the task first."
-        results = self.metric(
-            responses=self.outputs, references=self.dataset[self.label_column]
-        )
+        results = self.metric(self.outputs, self.labels)
         # logging.info(f"{self.name}:{results}")
         return results
 

tlem.py

@@ -82,6 +82,7 @@ class Suite(EvaluationSuite):
         "ceval-chat",
         "bbh",
         "drop",
+        "MATH",
     ]
 
     def __getitem__(self, key) -> Task:
@@ -215,6 +216,7 @@ class Suite(EvaluationSuite):
         except ValueError:
             logging.debug(f"add {task.name} to suite.")
             self.tasks.append(task)
+            logging.debug(self.tasks)
             return self.tasks[-1]
 
     def drop_duplicates(self, suite):