add math

tasks.py

@@ -1,8 +1,9 @@
 from dataclasses import dataclass, field
+
 from datasets import load_dataset, Dataset
 from functools import cached_property
 from tqdm.auto import tqdm
-from typing import Any, Optional, Protocol, Iterable, Callable
+from typing import Any, Optional, Callable
 import logging
 import pandas as pd
 from functools import partial
@@ -187,71 +188,57 @@ def multichoice_zh(responses: Any, references: list[str]):
 class Metrics:
     cmmlu = multichoice_zh
     mmlu = multichoice
-    
+
     def ceval(responses: list[str], answers: list[str | int]):
         responses = [extract_choice_zh(pred) for pred in responses]
         return responses, answers
-    
+
     def winogrande(responses: list[str], answers: list[str | int]):
         responses = [first_option_postprocess(pred, options="AB") for pred in responses]
         return responses, answers
-    
+
     def arc(responses: list[str], answers: list[str | int]):
         if len(responses) != len(answers):
-            return {
-                'error': 'predictions and references have different '
-                'length'
-            }
-        responses = [first_option_postprocess(pred, options="ABCD") for pred in responses]
+            return {"error": "predictions and references have different " "length"}
+        responses = [
+            first_option_postprocess(pred, options="ABCD") for pred in responses
+        ]
 
         return responses, answers
-    
+
     def hellaswag(responses: list[str], answers: list[str | int]):
         if len(responses) != len(answers):
-            return {
-                'error': 'predictions and references have different '
-                'length'
-            }
-        responses = [first_option_postprocess(pred, options="ABCD") for pred in responses]
-        answers = ['ABCD'[int(ans)] for ans in answers]
+            return {"error": "predictions and references have different " "length"}
+        responses = [
+            first_option_postprocess(pred, options="ABCD") for pred in responses
+        ]
+        answers = ["ABCD"[int(ans)] for ans in answers]
         return responses, answers
-    
+
     def drop(responses: list[str], answers: list[list]):
         if len(responses) != len(answers):
-            return {
-                'error': 'predictions and references have different '
-                'length'
-            }
+            return {"error": "predictions and references have different " "length"}
         responses = [general_postprocess(pred) for pred in responses]
-        processed_answers = [[general_postprocess(j) for j in i]
-                             for i in answers]
+        processed_answers = [[general_postprocess(j) for j in i] for i in answers]
         matched_answers = []
-        for pred, ans, origin_ans in zip(responses, processed_answers,
-                                         answers):
-            
+        for pred, ans, origin_ans in zip(responses, processed_answers, answers):
             if pred in ans or pred in origin_ans:
                 matched_answers.append(pred)
             else:
                 matched_answers.append(ans[0])
-            
+
         return responses, matched_answers
-    
+
     def bbh_mcq(responses: list[str], answers: list[str | int]):
         if len(responses) != len(answers):
-            return {
-                'error': 'predictions and references have different '
-                'length'
-            }
+            return {"error": "predictions and references have different " "length"}
         responses = [bbh_mcq_postprocess(pred) for pred in responses]
 
         return responses, answers
-    
+
     def bbh_freefrom(responses: list[str], answers: list[str | int]):
         if len(responses) != len(answers):
-            return {
-                'error': 'predictions and references have different '
-                'length'
-            }
+            return {"error": "predictions and references have different " "length"}
 
         responses = [bbh_freeform_postprocess(pred) for pred in responses]
 
@@ -272,27 +259,16 @@ class Metrics:
         return responses, answers
 
     def MATH(responses: list[str], answers: list[str]):
-        scores = []
-
-        for response, answer in zip(responses, answers):
+        extract_responses = []
+        for response in responses:
             indices = [pos for pos, char in enumerate(response) if char == "$"]
             if len(indices) <= 2:
-                scores.append(0)
-                continue
+                ans = ""
             else:
-                result = response[indices[-2] + 1 : indices[-1]]
-                gold = get_answer(answer)
-                scores.append(1.0 * is_equiv(result, gold))
-
-        return scores
-
-    def math23k(responses: list[str], answers: list[str]):
-        scores = []
-        for response, answer in zip(responses, answers):
-            pred = extract_numeric(response, pattern=NUMERIC_IN_ZH)
-            gold = extract_numeric(answer, pattern=NUMERIC_IN_ZH)
-            scores.append(1.0 * (pred == gold))
-        return scores
+                ans = response[indices[-2] + 1 : indices[-1]]
+            extract_responses.append(strip_string(ans))
+        extract_answers = [strip_string(get_answer(answer)) for answer in answers]
+        return extract_responses, extract_answers
 
 
 class CMMLU:
@@ -570,7 +546,7 @@ class MMLU:
 class Winogrande:
     input_column = "input"
     label_column = "answer"
-    
+
     categories = [
         "winogrande_debiased",
         "winogrande_l",
@@ -579,24 +555,24 @@ class Winogrande:
         "winogrande_xl",
         "winogrande_xs",
     ]
-    
+
     @classmethod
     def prompt_winogrande(cls, example):
-        option1 = example["sentence"].replace("_", example['option1'])
-        option2 = example["sentence"].replace("_", example['option2'])
+        option1 = example["sentence"].replace("_", example["option1"])
+        option2 = example["sentence"].replace("_", example["option2"])
         answer = example[cls.label_column]
         prompt = f"Which of the following is a good sentence:\nA. {option1}\nB. {option2}\nAnswer:"
-        
+
         return {
             cls.input_column: prompt,
-            cls.label_column: ' AB'[int(answer)] if answer != '' else ''
+            cls.label_column: " AB"[int(answer)] if answer != "" else "",
         }
-        
+
     @classmethod
-    def suite(cls,):
-        subcategories = {
-            item: [item] for item in cls.categories
-        }
+    def suite(
+        cls,
+    ):
+        subcategories = {item: [item] for item in cls.categories}
         finer_categories = (
             pd.Series(subcategories)  # noqa # type: ignore
             .explode()
@@ -618,19 +594,18 @@ class Winogrande:
                         label_column=cls.label_column,
                         prompt=partial(cls.prompt_winogrande),
                         few_shot=0,
-                        split="validation"
+                        split="validation",
                     )
                 )
-        
+
         return suite
-        
 
 
 class DROP:
     input_column = "input"
     label_column = "answers"
-    
-    icl_prompt = '''\
+
+    icl_prompt = """\
 Text: In the county, the population was spread out with 23.50% under the age of 18, 8.70% from 18 to 24, 29.70% from 25 to 44, 24.70% from 45 to 64, and 13.30% who were 65 years of age or older.
 Question: How many more percent are under the age of 18 compared to the 18 to 24 group?
 Anawer: According to the text, 23.5% are under the age of 18, and 8.7% are from ages 18 to 24. 23.5%-8.7%=14.8%. So the answer is 14.8.
@@ -641,15 +616,16 @@ Anawer: According to the text, Stafford threw 5 TD passes, 3 of which were to Jo
 
 Text: [PROMPT]
 Question: [QUESTION]
-Anawer:'''
-    
+Anawer:"""
+
     categories = ["validation"]
-    
+
     @classmethod
     def prompt_drop(cls, example):
-        
-        prompt = cls.icl_prompt.replace("[PROMPT]", example["passage"]).replace("[QUESTION]", example["question"])
-        
+        prompt = cls.icl_prompt.replace("[PROMPT]", example["passage"]).replace(
+            "[QUESTION]", example["question"]
+        )
+
         validated_answers = example["answers_spans"]["spans"]
         validated_types = example["answers_spans"]["types"]
         answers = []
@@ -661,18 +637,16 @@ Anawer:'''
             #     answers.append(' '.join(d).strip())
             # else:
             #     for span in answer_item['spans']:
-                    # answers.append(span)
+            # answers.append(span)
             answers.append(answer_item)
         answers = list(set(answers))
-        
-        return {
-            cls.input_column: prompt,
-            cls.label_column: answers
-        }
-    
+
+        return {cls.input_column: prompt, cls.label_column: answers}
+
     @classmethod
-    def suite(cls,):
-        
+    def suite(
+        cls,
+    ):
         finer_categories = (
             pd.Series(cls.categories)  # noqa # type: ignore
             .explode()
@@ -693,33 +667,34 @@ Anawer:'''
                     label_column=cls.label_column,
                     prompt=partial(cls.prompt_drop),
                     few_shot=0,
-                    split="validation"
+                    split="validation",
                 )
             )
-        
+
         return suite
 
 
 class HellaSwag:
     input_column = "input"
     label_column = "label"
-    
+
     categories = ["validation"]
-    
+
     @classmethod
     def prompt_hellaswag(cls, example):
-        
         prompt = f"{example['ctx']}\nQuestion: Which ending makes the most sense?\n"
         prompt += f"A. {example['endings'][0]}\n"
         prompt += f"B. {example['endings'][1]}\n"
         prompt += f"C. {example['endings'][2]}\n"
         prompt += f"D. {example['endings'][3]}\n"
         prompt += "You may choose from 'A', 'B', 'C', 'D'.\nAnswer:"
-        
+
         return {cls.input_column: prompt}
-    
+
     @classmethod
-    def suite(cls,):
+    def suite(
+        cls,
+    ):
         finer_categories = (
             pd.Series(cls.categories)  # noqa # type: ignore
             .explode()
@@ -740,21 +715,22 @@ class HellaSwag:
                     label_column=cls.label_column,
                     prompt=partial(cls.prompt_hellaswag),
                     few_shot=0,
-                    split="validation"
+                    split="validation",
                 )
             )
-        
+
         return suite
 
+
 class ARC:
     input_column = "input"
     label_column = "answerKey"
-    
+
     categories = [
         "ARC-Challenge",
         "ARC-Easy",
     ]
-    
+
     @classmethod
     def prompt_arc(cls, example):
         choices = example["choices"]
@@ -762,10 +738,8 @@ class ARC:
         for label, choice in zip(choices["label"], choices["text"]):
             prompt += f"\n{label}. {choice}"
         prompt += "\nAnswer:"
-        return {
-            cls.input_column: prompt
-        }
-    
+        return {cls.input_column: prompt}
+
     @classmethod
     def suite(cls):
         finer_categories = (
@@ -790,62 +764,71 @@ class ARC:
                     few_shot=0,
                 )
             )
-        
+
         return suite
 
 
 class BBH:
     input_column = "input"
     label_column = "target"
-    
+
     multiple_choice_prefix = "Follow the given examples and answer the question.\n[HINT]\n\nQ: [INPUT]\nA: Let's think step by step."
     free_form_prefix = "Follow the given examples and answer the question.\n[HINT]\n\nQ: [INPUT]\nA: Let's think step by step."
-    
+
     bbh_multiple_choice_sets = [
-        'temporal_sequences',
-        'disambiguation_qa',
-        'date_understanding',
-        'tracking_shuffled_objects_three_objects',
-        'penguins_in_a_table',
-        'geometric_shapes',
-        'snarks',
-        'ruin_names',
-        'tracking_shuffled_objects_seven_objects',
-        'tracking_shuffled_objects_five_objects',
-        'logical_deduction_three_objects',
-        'hyperbaton',
-        'logical_deduction_five_objects',
-        'logical_deduction_seven_objects',
-        'movie_recommendation',
-        'salient_translation_error_detection',
-        'reasoning_about_colored_objects',
+        "temporal_sequences",
+        "disambiguation_qa",
+        "date_understanding",
+        "tracking_shuffled_objects_three_objects",
+        "penguins_in_a_table",
+        "geometric_shapes",
+        "snarks",
+        "ruin_names",
+        "tracking_shuffled_objects_seven_objects",
+        "tracking_shuffled_objects_five_objects",
+        "logical_deduction_three_objects",
+        "hyperbaton",
+        "logical_deduction_five_objects",
+        "logical_deduction_seven_objects",
+        "movie_recommendation",
+        "salient_translation_error_detection",
+        "reasoning_about_colored_objects",
     ]
-    
+
     bbh_free_form_sets = [
-        'multistep_arithmetic_two',
-        'navigate',
-        'dyck_languages',
-        'word_sorting',
-        'sports_understanding',
-        'boolean_expressions',
-        'object_counting',
-        'formal_fallacies',
-        'causal_judgement',
-        'web_of_lies',
+        "multistep_arithmetic_two",
+        "navigate",
+        "dyck_languages",
+        "word_sorting",
+        "sports_understanding",
+        "boolean_expressions",
+        "object_counting",
+        "formal_fallacies",
+        "causal_judgement",
+        "web_of_lies",
     ]
-    
+
     @classmethod
-    def prompt_bbh(cls, example, category:str):
-        
-        meta_prompt = cls.multiple_choice_prefix if category in cls.bbh_multiple_choice_sets else cls.free_form_prefix
-        prompt = meta_prompt.replace("[HINT]", bbh_lib_prompt(category=category)).replace("[INPUT]", example[cls.input_column])
-        
+    def prompt_bbh(cls, example, category: str):
+        meta_prompt = (
+            cls.multiple_choice_prefix
+            if category in cls.bbh_multiple_choice_sets
+            else cls.free_form_prefix
+        )
+        prompt = meta_prompt.replace(
+            "[HINT]", bbh_lib_prompt(category=category)
+        ).replace("[INPUT]", example[cls.input_column])
+
         return {"input": prompt}
-    
+
     @classmethod
-    def suite(cls,):
+    def suite(
+        cls,
+    ):
         finer_categories = (
-            pd.Series(cls.bbh_free_form_sets + cls.bbh_multiple_choice_sets)  # noqa # type: ignore
+            pd.Series(
+                cls.bbh_free_form_sets + cls.bbh_multiple_choice_sets
+            )  # noqa # type: ignore
             .explode()
             .reset_index()
             .set_index(0)
@@ -878,167 +861,229 @@ class BBH:
                         few_shot=0,
                     )
                 )
-        
+
         return suite
-    
 
-    
+
 class CEVAL:
     input_column = "input"
     label_column = "answer"
-    
+
     @classmethod
-    def prompt_ceval(cls, example, cate:str, chat=False):
+    def prompt_ceval(cls, example, cate: str, chat=False):
         _ch_name = cls.ceval_subject_mapping[cate][1]
-        prefix = (
-            f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案��\n"
-            if chat
-            else "问题："
-        )
-        
+        prefix = f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n" if chat else "问题："
+
         prompt = prefix + f'{example["question"]}'
         for choice in list("ABCD"):
             prompt += f"\n{choice}. {example[choice]}"
 
         prompt += "\n答案："
         return {"input": prompt}
-    
+
     ceval_subject_mapping = {
-        "computer_network":
-        ["Computer Network", "\u8ba1\u7b97\u673a\u7f51\u7edc", "STEM"],
-        "operating_system":
-        ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"],
-        "computer_architecture":
-        ["Computer Architecture", "\u8ba1\u7b97\u673a\u7ec4\u6210", "STEM"],
-        "college_programming":
-        ["College Programming", "\u5927\u5b66\u7f16\u7a0b", "STEM"],
+        "computer_network": [
+            "Computer Network",
+            "\u8ba1\u7b97\u673a\u7f51\u7edc",
+            "STEM",
+        ],
+        "operating_system": ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"],
+        "computer_architecture": [
+            "Computer Architecture",
+            "\u8ba1\u7b97\u673a\u7ec4\u6210",
+            "STEM",
+        ],
+        "college_programming": [
+            "College Programming",
+            "\u5927\u5b66\u7f16\u7a0b",
+            "STEM",
+        ],
         "college_physics": ["College Physics", "\u5927\u5b66\u7269\u7406", "STEM"],
-        "college_chemistry":
-        ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"],
-        "advanced_mathematics":
-        ["Advanced Mathematics", "\u9ad8\u7b49\u6570\u5b66", "STEM"],
-        "probability_and_statistics":
-        ["Probability and Statistics", "\u6982\u7387\u7edf\u8ba1", "STEM"],
-        "discrete_mathematics":
-        ["Discrete Mathematics", "\u79bb\u6563\u6570\u5b66", "STEM"],
+        "college_chemistry": ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"],
+        "advanced_mathematics": [
+            "Advanced Mathematics",
+            "\u9ad8\u7b49\u6570\u5b66",
+            "STEM",
+        ],
+        "probability_and_statistics": [
+            "Probability and Statistics",
+            "\u6982\u7387\u7edf\u8ba1",
+            "STEM",
+        ],
+        "discrete_mathematics": [
+            "Discrete Mathematics",
+            "\u79bb\u6563\u6570\u5b66",
+            "STEM",
+        ],
         "electrical_engineer": [
-            "Electrical Engineer", "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08",
-            "STEM"
+            "Electrical Engineer",
+            "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08",
+            "STEM",
+        ],
+        "metrology_engineer": [
+            "Metrology Engineer",
+            "\u6ce8\u518c\u8ba1\u91cf\u5e08",
+            "STEM",
+        ],
+        "high_school_mathematics": [
+            "High School Mathematics",
+            "\u9ad8\u4e2d\u6570\u5b66",
+            "STEM",
+        ],
+        "high_school_physics": [
+            "High School Physics",
+            "\u9ad8\u4e2d\u7269\u7406",
+            "STEM",
+        ],
+        "high_school_chemistry": [
+            "High School Chemistry",
+            "\u9ad8\u4e2d\u5316\u5b66",
+            "STEM",
         ],
-        "metrology_engineer":
-        ["Metrology Engineer", "\u6ce8\u518c\u8ba1\u91cf\u5e08", "STEM"],
-        "high_school_mathematics":
-        ["High School Mathematics", "\u9ad8\u4e2d\u6570\u5b66", "STEM"],
-        "high_school_physics":
-        ["High School Physics", "\u9ad8\u4e2d\u7269\u7406", "STEM"],
-        "high_school_chemistry":
-        ["High School Chemistry", "\u9ad8\u4e2d\u5316\u5b66", "STEM"],
         "high_school_biology": [
-            "High School Biology", "\u9ad8\u4e2d\u751f\u7269", "STEM"
+            "High School Biology",
+            "\u9ad8\u4e2d\u751f\u7269",
+            "STEM",
         ],
         "middle_school_mathematics": [
-            "Middle School Mathematics", "\u521d\u4e2d\u6570\u5b66", "STEM"
+            "Middle School Mathematics",
+            "\u521d\u4e2d\u6570\u5b66",
+            "STEM",
         ],
         "middle_school_biology": [
-            "Middle School Biology", "\u521d\u4e2d\u751f\u7269", "STEM"
+            "Middle School Biology",
+            "\u521d\u4e2d\u751f\u7269",
+            "STEM",
         ],
         "middle_school_physics": [
-            "Middle School Physics", "\u521d\u4e2d\u7269\u7406", "STEM"
+            "Middle School Physics",
+            "\u521d\u4e2d\u7269\u7406",
+            "STEM",
         ],
         "middle_school_chemistry": [
-            "Middle School Chemistry", "\u521d\u4e2d\u5316\u5b66", "STEM"
-        ],
-        "veterinary_medicine": [
-            "Veterinary Medicine", "\u517d\u533b\u5b66", "STEM"
+            "Middle School Chemistry",
+            "\u521d\u4e2d\u5316\u5b66",
+            "STEM",
         ],
+        "veterinary_medicine": ["Veterinary Medicine", "\u517d\u533b\u5b66", "STEM"],
         "college_economics": [
-            "College Economics", "\u5927\u5b66\u7ecf\u6d4e\u5b66", "Social Science"
+            "College Economics",
+            "\u5927\u5b66\u7ecf\u6d4e\u5b66",
+            "Social Science",
         ],
         "business_administration": [
-            "Business Administration", "\u5de5\u5546\u7ba1\u7406", "Social Science"
+            "Business Administration",
+            "\u5de5\u5546\u7ba1\u7406",
+            "Social Science",
         ],
         "marxism": [
-            "Marxism", "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406",
-            "Social Science"
+            "Marxism",
+            "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406",
+            "Social Science",
         ],
         "mao_zedong_thought": [
             "Mao Zedong Thought",
             "\u6bdb\u6cfd\u4e1c\u601d\u60f3\u548c\u4e2d\u56fd\u7279\u8272\u793e\u4f1a\u4e3b\u4e49\u7406\u8bba\u4f53\u7cfb\u6982\u8bba",
-            "Social Science"
+            "Social Science",
         ],
         "education_science": [
-            "Education Science", "\u6559\u80b2\u5b66", "Social Science"
+            "Education Science",
+            "\u6559\u80b2\u5b66",
+            "Social Science",
         ],
         "teacher_qualification": [
-            "Teacher Qualification", "\u6559\u5e08\u8d44\u683c", "Social Science"
+            "Teacher Qualification",
+            "\u6559\u5e08\u8d44\u683c",
+            "Social Science",
         ],
         "high_school_politics": [
-            "High School Politics", "\u9ad8\u4e2d\u653f\u6cbb", "Social Science"
+            "High School Politics",
+            "\u9ad8\u4e2d\u653f\u6cbb",
+            "Social Science",
         ],
         "high_school_geography": [
-            "High School Geography", "\u9ad8\u4e2d\u5730\u7406", "Social Science"
+            "High School Geography",
+            "\u9ad8\u4e2d\u5730\u7406",
+            "Social Science",
         ],
         "middle_school_politics": [
-            "Middle School Politics", "\u521d\u4e2d\u653f\u6cbb", "Social Science"
+            "Middle School Politics",
+            "\u521d\u4e2d\u653f\u6cbb",
+            "Social Science",
         ],
         "middle_school_geography": [
-            "Middle School Geography", "\u521d\u4e2d\u5730\u7406", "Social Science"
+            "Middle School Geography",
+            "\u521d\u4e2d\u5730\u7406",
+            "Social Science",
+        ],
+        "modern_chinese_history": [
+            "Modern Chinese History",
+            "\u8fd1\u4ee3\u53f2\u7eb2\u8981",
+            "Humanities",
         ],
-        "modern_chinese_history":
-        ["Modern Chinese History", "\u8fd1\u4ee3\u53f2\u7eb2\u8981", "Humanities"],
         "ideological_and_moral_cultivation": [
             "Ideological and Moral Cultivation",
             "\u601d\u60f3\u9053\u5fb7\u4fee\u517b\u4e0e\u6cd5\u5f8b\u57fa\u7840",
-            "Humanities"
+            "Humanities",
         ],
         "logic": ["Logic", "\u903b\u8f91\u5b66", "Humanities"],
         "law": ["Law", "\u6cd5\u5b66", "Humanities"],
         "chinese_language_and_literature": [
             "Chinese Language and Literature",
-            "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66", "Humanities"
+            "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66",
+            "Humanities",
         ],
         "art_studies": ["Art Studies", "\u827a\u672f\u5b66", "Humanities"],
         "professional_tour_guide": [
-            "Professional Tour Guide", "\u5bfc\u6e38\u8d44\u683c", "Humanities"
+            "Professional Tour Guide",
+            "\u5bfc\u6e38\u8d44\u683c",
+            "Humanities",
         ],
         "legal_professional": [
-            "Legal Professional", "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c",
-            "Humanities"
+            "Legal Professional",
+            "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c",
+            "Humanities",
         ],
         "high_school_chinese": [
-            "High School Chinese", "\u9ad8\u4e2d\u8bed\u6587", "Humanities"
+            "High School Chinese",
+            "\u9ad8\u4e2d\u8bed\u6587",
+            "Humanities",
         ],
         "high_school_history": [
-            "High School History", "\u9ad8\u4e2d\u5386\u53f2", "Humanities"
+            "High School History",
+            "\u9ad8\u4e2d\u5386\u53f2",
+            "Humanities",
         ],
         "middle_school_history": [
-            "Middle School History", "\u521d\u4e2d\u5386\u53f2", "Humanities"
+            "Middle School History",
+            "\u521d\u4e2d\u5386\u53f2",
+            "Humanities",
         ],
         "civil_servant": ["Civil Servant", "\u516c\u52a1\u5458", "Other"],
         "sports_science": ["Sports Science", "\u4f53\u80b2\u5b66", "Other"],
-        "plant_protection": [
-            "Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other"
-        ],
+        "plant_protection": ["Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other"],
         "basic_medicine": ["Basic Medicine", "\u57fa\u7840\u533b\u5b66", "Other"],
-        "clinical_medicine": [
-            "Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other"
-        ],
+        "clinical_medicine": ["Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other"],
         "urban_and_rural_planner": [
             "Urban and Rural Planner",
-            "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08", "Other"
+            "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08",
+            "Other",
         ],
         "accountant": ["Accountant", "\u6ce8\u518c\u4f1a\u8ba1\u5e08", "Other"],
         "fire_engineer": [
-            "Fire Engineer", "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08", "Other"
+            "Fire Engineer",
+            "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08",
+            "Other",
         ],
         "environmental_impact_assessment_engineer": [
             "Environmental Impact Assessment Engineer",
-            "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08", "Other"
+            "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08",
+            "Other",
         ],
         "tax_accountant": ["Tax Accountant", "\u7a0e\u52a1\u5e08", "Other"],
-        "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"]
+        "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"],
     }
-    
+
     @classmethod
     def suite(cls, chat: bool):
         suite = defaultdict(list)
@@ -1058,8 +1103,8 @@ class CEVAL:
                         prompt=partial(cls.prompt_ceval, cate=subject, chat=chat),
                         few_shot=0 if chat else 5,
                         few_shot_from="dev",
-                        split="val"
+                        split="val",
                     )
                 )
-                
-        return suite
+
+        return suite

tlem.py

@@ -135,6 +135,15 @@ class Suite(EvaluationSuite):
                     prompt=mt_bench_prompt
                     # metric_name=("sustech/tlem", "gsm8k"),
                 )
+            case "MATH" | "competition_math":
+                suite = Task(
+                    dataset_name="hendrycks/competition_math",
+                    split="test",
+                    prompt="This is a math problem, please think step by step and slove it: {input_column}",
+                    metric_name=("sustech/tlem", "MATH"),
+                    input_column="problem",
+                    label_column="solution",
+                )
         match name:
             case _ if "test" in name:
                 suite = suite["Test"]