FIX: extraction func of C-Eval; logging metrics (#3)

- Delete: print extraction info (868c1b24af35fcd7a5a09f382e15d6e794a7b5d7)
- FIX: extraction func of C-Eval; logging metrics (e637e0c3ca817b49543ee6615f52d7c4cc7158fa)


Co-authored-by: songzhuoyang <Cookize@users.noreply.huggingface.co>

tasks.py

@@ -149,14 +149,15 @@ class Task:
             return
         self.outputs = outputs
         try:
-            result = self.metric._compute(
-                responses=outputs, references=self.dataset[self.label_column]
-            )
+            try:
+                result = self.metric._compute(
+                    responses=outputs, references=self.dataset[self.label_column]
+                )
+            except Exception as e:
+                result = self.metric.compute(
+                    responses=outputs, references=self.dataset[self.label_column]
+                )
         except Exception as e:
-            result = self.metric.compute(
-                responses=outputs, references=self.dataset[self.label_column]
-            )
-        finally:
             result = outputs
         # if log:
         #     name = name or pipeline.__name__
@@ -188,7 +189,7 @@ class Metrics:
     mmlu = multichoice
     
     def ceval(responses: list[str], answers: list[str | int]):
-        responses = [first_capital_postprocess(pred) for pred in responses]
+        responses = [extract_choice_zh(pred) for pred in responses]
         return responses, answers
     
     def winogrande(responses: list[str], answers: list[str | int]):
@@ -892,7 +893,7 @@ class CEVAL:
         prefix = (
             f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n"
             if chat
-            else "问题"
+            else "问题："
         )
         
         prompt = prefix + f'{example["question"]}'
@@ -1043,6 +1044,7 @@ class CEVAL:
         suite = defaultdict(list)
         cls.categories = defaultdict(list)
         for task, info in cls.ceval_subject_mapping.items():
+            cls.categories[info[0]].append(task)
             cls.categories[info[2]].append(task)
         cls.categories["all"] = list(cls.ceval_subject_mapping.keys())
         for k, v in cls.categories.items():

tlem.py

@@ -58,7 +58,7 @@ class ReasoningMetric(evaluate.Metric):
         )
         df["extract_responses"] = extract_responses
         df["extract_references"] = extract_references
-        print(df)
+        # print(df)
         results = {
             "Accuracy": (df["extract_references"] == df["extract_responses"])
             .astype(int)
@@ -139,7 +139,7 @@ class Suite(EvaluationSuite):
             case _ if "test" in name:
                 suite = suite["Test"]
 
-        self.suite = suite
+        self.suite = [suite] if isinstance(suite, Task) else suite
 
     def __init__(self, name="tlem"):
         super().__init__(name)