|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| import os
|
|
|
| import datasets
|
| import pandas as pd
|
|
|
|
|
| _CITATION = """\
|
| @article{huang2023ceval,
|
| title={C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models},
|
| author={Huang, Yuzhen and Bai, Yuzhuo and Zhu, Zhihao and others},
|
| journal={arXiv preprint arXiv:2305.08322},
|
| year={2023}
|
| }
|
| """
|
|
|
| _DESCRIPTION = """\
|
| C-Eval is a comprehensive Chinese evaluation suite for foundation models.
|
| It consists of 13948 multi-choice questions spanning 52 diverse disciplines and four difficulty levels.
|
| """
|
|
|
| _HOMEPAGE = "https://cevalbenchmark.com"
|
|
|
| _LICENSE = "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License"
|
|
|
| _URL = "ceval.zip"
|
|
|
| task_list = [
|
| "computer_network",
|
| "operating_system",
|
| "computer_architecture",
|
| "college_programming",
|
| "college_physics",
|
| "college_chemistry",
|
| "advanced_mathematics",
|
| "probability_and_statistics",
|
| "discrete_mathematics",
|
| "electrical_engineer",
|
| "metrology_engineer",
|
| "high_school_mathematics",
|
| "high_school_physics",
|
| "high_school_chemistry",
|
| "high_school_biology",
|
| "middle_school_mathematics",
|
| "middle_school_biology",
|
| "middle_school_physics",
|
| "middle_school_chemistry",
|
| "veterinary_medicine",
|
| "college_economics",
|
| "business_administration",
|
| "marxism",
|
| "mao_zedong_thought",
|
| "education_science",
|
| "teacher_qualification",
|
| "high_school_politics",
|
| "high_school_geography",
|
| "middle_school_politics",
|
| "middle_school_geography",
|
| "modern_chinese_history",
|
| "ideological_and_moral_cultivation",
|
| "logic",
|
| "law",
|
| "chinese_language_and_literature",
|
| "art_studies",
|
| "professional_tour_guide",
|
| "legal_professional",
|
| "high_school_chinese",
|
| "high_school_history",
|
| "middle_school_history",
|
| "civil_servant",
|
| "sports_science",
|
| "plant_protection",
|
| "basic_medicine",
|
| "clinical_medicine",
|
| "urban_and_rural_planner",
|
| "accountant",
|
| "fire_engineer",
|
| "environmental_impact_assessment_engineer",
|
| "tax_accountant",
|
| "physician",
|
| ]
|
|
|
|
|
| class CevalConfig(datasets.BuilderConfig):
|
| def __init__(self, **kwargs):
|
| super().__init__(version=datasets.Version("1.0.0"), **kwargs)
|
|
|
|
|
| class Ceval(datasets.GeneratorBasedBuilder):
|
| BUILDER_CONFIGS = [
|
| CevalConfig(
|
| name=task_name,
|
| )
|
| for task_name in task_list
|
| ]
|
|
|
| def _info(self):
|
| features = datasets.Features(
|
| {
|
| "id": datasets.Value("int32"),
|
| "question": datasets.Value("string"),
|
| "A": datasets.Value("string"),
|
| "B": datasets.Value("string"),
|
| "C": datasets.Value("string"),
|
| "D": datasets.Value("string"),
|
| "answer": datasets.Value("string"),
|
| "explanation": datasets.Value("string"),
|
| }
|
| )
|
| return datasets.DatasetInfo(
|
| description=_DESCRIPTION,
|
| features=features,
|
| homepage=_HOMEPAGE,
|
| license=_LICENSE,
|
| citation=_CITATION,
|
| )
|
|
|
| def _split_generators(self, dl_manager):
|
| data_dir = dl_manager.download_and_extract(_URL)
|
| task_name = self.config.name
|
| return [
|
| datasets.SplitGenerator(
|
| name=datasets.Split.TEST,
|
| gen_kwargs={
|
| "filepath": os.path.join(data_dir, "test", f"{task_name}_test.csv"),
|
| },
|
| ),
|
| datasets.SplitGenerator(
|
| name=datasets.Split.VALIDATION,
|
| gen_kwargs={
|
| "filepath": os.path.join(data_dir, "val", f"{task_name}_val.csv"),
|
| },
|
| ),
|
| datasets.SplitGenerator(
|
| name=datasets.Split.TRAIN,
|
| gen_kwargs={
|
| "filepath": os.path.join(data_dir, "dev", f"{task_name}_dev.csv"),
|
| },
|
| ),
|
| ]
|
|
|
| def _generate_examples(self, filepath):
|
| df = pd.read_csv(filepath, encoding="utf-8")
|
| for i, instance in enumerate(df.to_dict(orient="records")):
|
| if "answer" not in instance.keys():
|
| instance["answer"] = ""
|
| if "explanation" not in instance.keys():
|
| instance["explanation"] = ""
|
| yield i, instance
|
|
|
