Add files using upload-large-folder tool

build/lib/opencompass/configs/chatml_datasets/AMO_Bench/AMO_Bench_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='AMO-Bench',
+        path='./data/amo-bench.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/CPsyExam/CPsyExam_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='CPsyExam',
+        path='./data/CPsyExam/merged_train_dev.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/CS_Bench/CS_Bench_gen.py

@@ -0,0 +1,25 @@
+
+subset_list = [
+    'test',
+    'valid',
+]
+
+language_list = [
+    'CN',
+    'EN',
+]
+
+datasets = []
+
+for subset in subset_list:
+    for language in language_list:
+        datasets.append(
+            dict(
+                abbr=f'CS-Bench_{language}_{subset}',
+                path=f'./data/csbench/CSBench-{language}/{subset}.jsonl',
+                evaluator=dict(
+                    type='llm_evaluator',
+                    judge_cfg=dict(),
+                ),
+            )
+        )

build/lib/opencompass/configs/chatml_datasets/C_MHChem/C_MHChem_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='C-MHChem',
+        path='./data/C-MHChem2.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/HMMT2025/HMMT2025_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='HMMT2025',
+        path='./data/hmmt2025.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/IMO_Bench_AnswerBench/IMO_Bench_AnswerBench_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='IMO-Bench-AnswerBench',
+        path='./data/imo-bench-answerbench.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/MaScQA/MaScQA_gen.py

@@ -0,0 +1,12 @@
+
+datasets = [
+    dict(
+        abbr='MaScQA',
+        path='./data/MaScQA/MaScQA.jsonl',
+        evaluator=dict(
+            type='llm_evaluator',
+            judge_cfg=dict(),
+        ),
+        n=1,
+    ),
+]

build/lib/opencompass/configs/chatml_datasets/UGPhysics/UGPhysics_gen.py

@@ -0,0 +1,36 @@
+
+subset_list = [
+    'AtomicPhysics',
+    'ClassicalElectromagnetism',
+    'ClassicalMechanics',
+    'Electrodynamics',
+    'GeometricalOptics',
+    'QuantumMechanics',
+    'Relativity',
+    'Solid-StatePhysics',
+    'StatisticalMechanics',
+    'SemiconductorPhysics',
+    'Thermodynamics',
+    'TheoreticalMechanics',
+    'WaveOptics',
+]
+
+language_list = [
+    'zh',
+    'en',
+]
+
+datasets = []
+
+for subset in subset_list:
+    for language in language_list:
+        datasets.append(
+            dict(
+                abbr=f'UGPhysics_{subset}_{language}',
+                path=f'./data/ugphysics/{subset}/{language}.jsonl',
+                evaluator=dict(
+                    type='llm_evaluator',
+                    judge_cfg=dict(),
+                ),
+            )
+        )

build/lib/opencompass/configs/datasets/ARC_Prize_Public_Evaluation/README.md

@@ -0,0 +1,47 @@
+# ARC Prize Public Evaluation
+
+#### Overview
+The spirit of ARC Prize is to open source progress towards AGI. To win prize money, you will be required to publish reproducible code/methods into public domain.
+
+ARC Prize measures AGI progress using the [ARC-AGI private evaluation set](https://arcprize.org/guide#private), [the leaderboard is here](https://arcprize.org/leaderboard), and the Grand Prize is unlocked once the first team reaches [at least 85%](https://arcprize.org/guide#grand-prize-goal).
+
+Note: the private evaluation set imposes limitations on solutions (eg. no internet access, so no GPT-4/Claude/etc). There is a [secondary leaderboard](https://arcprize.org/leaderboard) called ARC-AGI-Pub, it measures the [public evaluation set](https://arcprize.org/guide#public-tasks) and imposes no limits but it is not part of ARC Prize 2024 at this time.
+
+
+#### Tasks
+ARC-AGI tasks are a series of three to five input and output tasks followed by a final task with only the input listed. Each task tests the utilization of a specific learned skill based on a minimal number of cognitive priors.
+
+![alt text](https://arcprize.org/media/images/arc-task-grids.jpg)
+
+Tasks are represented as JSON lists of integers. These JSON objects can also be represented visually as a grid of colors using an ARC-AGI task viewer.
+
+A successful submission is a pixel-perfect description (color and position) of the final task's output.
+
+#### Format
+
+As mentioned above, tasks are stored in JSON format. Each JSON file consists of two key-value pairs.
+
+`train`: a list of two to ten input/output pairs (typically three.) These are used for your algorithm to infer a rule.
+
+`test`: a list of one to three input/output pairs (typically one.) Your model should apply the inferred rule from the train set and construct an output solution. You will have access to the output test solution on the public data. The output solution on the private evaluation set will not be revealed.
+
+Here is an example of a simple ARC-AGI task that has three training pairs along with a single test pair. Each pair is shown as a 2x2 grid. There are four colors represented by the integers 1, 4, 6, and 8. Which actual color (red/green/blue/black) is applied to each integer is arbitrary and up to you.
+
+```json
+{
+  "train": [
+    {"input": [[1, 0], [0, 0]], "output": [[1, 1], [1, 1]]},
+    {"input": [[0, 0], [4, 0]], "output": [[4, 4], [4, 4]]},
+    {"input": [[0, 0], [6, 0]], "output": [[6, 6], [6, 6]]}
+  ],
+  "test": [
+    {"input": [[0, 0], [0, 8]], "output": [[8, 8], [8, 8]]}
+  ]
+}
+```
+
+#### Performance
+
+| Qwen2.5-72B-Instruct | LLaMA3.1-70B-Instruct | gemma-2-27b-it | 
+| ----- | ----- |  ----- | 
+| 0.09 | 0.06 | 0.05 |

build/lib/opencompass/configs/datasets/ARC_Prize_Public_Evaluation/arc_prize_public_evaluation_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .arc_prize_public_evaluation_gen_872059 import arc_prize_public_evaluation_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/ARC_Prize_Public_Evaluation/arc_prize_public_evaluation_gen_872059.py

@@ -0,0 +1,56 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.arc_prize_public_evaluation import ARCPrizeDataset, ARCPrizeEvaluator
+
+
+# The system_prompt defines the initial instructions for the model, 
+# setting the context for solving ARC tasks.
+system_prompt = '''You are a puzzle solving wizard. You are given a puzzle from the abstraction and reasoning corpus developed by Francois Chollet.'''
+
+# User message template is a template for creating user prompts. It includes placeholders for training data and test input data, 
+# guiding the model to learn the rule and apply it to solve the given puzzle.
+user_message_template = '''Here are the example input and output pairs from which you should learn the underlying rule to later predict the output for the given test input:
+----------------------------------------
+{training_data}
+----------------------------------------
+Now, solve the following puzzle based on its input grid by applying the rules you have learned from the training data.:
+----------------------------------------
+[{{'input': {input_test_data}, 'output': [[]]}}]
+----------------------------------------
+What is the output grid? Only provide the output grid in the form as in the example input and output pairs. Do not provide any additional information:'''
+
+
+arc_prize_public_evaluation_reader_cfg = dict(
+    input_columns=['training_data', 'input_test_data'], 
+    output_column='output_test_data'
+)
+
+arc_prize_public_evaluation_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='SYSTEM', prompt=system_prompt),
+                dict(role='HUMAN', prompt=user_message_template),
+            ],
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=2048)
+)
+
+arc_prize_public_evaluation_eval_cfg = dict(
+    evaluator=dict(type=ARCPrizeEvaluator)
+)
+
+arc_prize_public_evaluation_datasets = [
+    dict(
+        abbr='ARC_Prize_Public_Evaluation',
+        type=ARCPrizeDataset,
+        path='opencompass/arc_prize_public_evaluation',
+        reader_cfg=arc_prize_public_evaluation_reader_cfg,
+        infer_cfg=arc_prize_public_evaluation_infer_cfg,
+        eval_cfg=arc_prize_public_evaluation_eval_cfg
+    )
+]

build/lib/opencompass/configs/datasets/ARC_Prize_Public_Evaluation/arc_prize_public_evaluation_gen_fedd04.py

@@ -0,0 +1,56 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.arc_prize_public_evaluation import ARCPrizeDataset, ARCPrizeEvaluator
+
+
+# The system_prompt defines the initial instructions for the model, 
+# setting the context for solving ARC tasks.
+system_prompt = '''You are a puzzle solving wizard. You are given a puzzle from the abstraction and reasoning corpus developed by Francois Chollet.'''
+
+# User message template is a template for creating user prompts. It includes placeholders for training data and test input data, 
+# guiding the model to learn the rule and apply it to solve the given puzzle.
+user_message_template = '''Here are the example input and output pairs from which you should learn the underlying rule to later predict the output for the given test input:
+----------------------------------------
+{training_data}
+----------------------------------------
+Now, solve the following puzzle based on its input grid by applying the rules you have learned from the training data.:
+----------------------------------------
+[{{'input': {input_test_data}, 'output': [[]]}}]
+----------------------------------------
+What is the output grid? Only provide the output grid in the form as in the example input and output pairs. Do not provide any additional information:'''
+
+
+arc_prize_public_evaluation_reader_cfg = dict(
+    input_columns=['training_data', 'input_test_data'], 
+    output_column='output_test_data'
+)
+
+arc_prize_public_evaluation_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='SYSTEM',fallback_role='HUMAN', prompt=system_prompt),
+                dict(role='HUMAN', prompt=user_message_template),
+            ],
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer)
+)
+
+arc_prize_public_evaluation_eval_cfg = dict(
+    evaluator=dict(type=ARCPrizeEvaluator)
+)
+
+arc_prize_public_evaluation_datasets = [
+    dict(
+        abbr='ARC_Prize_Public_Evaluation',
+        type=ARCPrizeDataset,
+        path='opencompass/arc_prize_public_evaluation',
+        reader_cfg=arc_prize_public_evaluation_reader_cfg,
+        infer_cfg=arc_prize_public_evaluation_infer_cfg,
+        eval_cfg=arc_prize_public_evaluation_eval_cfg
+    )
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_clean_ppl.py

@@ -0,0 +1,55 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccContaminationEvaluator
+from opencompass.datasets import ARCDatasetClean as ARCDataset
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
+                ], ),
+            'B':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
+                ], ),
+            'C':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
+                ], ),
+            'D':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
+                ], ),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_c_eval_cfg = dict(evaluator=dict(type=AccContaminationEvaluator),
+                      analyze_contamination=True)
+
+ARC_c_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-c-test',
+        path='opencompass/ai2_arc-test',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_cot_gen_926652.py

@@ -0,0 +1,53 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess, match_answer_pattern
+
+QUERY_TEMPLATE = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
+
+{question}
+
+A. {textA}
+B. {textB}
+C. {textC}
+D. {textD}
+""".strip()
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=QUERY_TEMPLATE)
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ARC_c_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+)
+
+ARC_c_datasets = [
+    dict(
+        abbr='ARC-c',
+        type=ARCDataset,
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_few_shot_gen_e9b043.py

@@ -0,0 +1,48 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever, FixKRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey',
+)
+
+ARC_c_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin='</E>',
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:',
+                ),
+                dict(role='BOT', prompt='{answerKey}'),
+            ],
+        ),
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
+    inferencer=dict(type=GenInferencer, max_out_len=50),
+)
+
+ARC_c_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+ARC_c_datasets = [
+    dict(
+        abbr='ARC-c',
+        type=ARCDataset,
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_few_shot_ppl.py

@@ -0,0 +1,63 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever, FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey',
+)
+
+ARC_c_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template={
+            'A': dict(
+                begin='</E>',
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}'),
+                ],
+            ),
+            'B': dict(
+                begin='</E>',
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}'),
+                ],
+            ),
+            'C': dict(
+                begin='</E>',
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}'),
+                ],
+            ),
+            'D': dict(
+                begin='</E>',
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}'),
+                ],
+            ),
+        },
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
+    inferencer=dict(type=PPLInferencer),
+)
+
+ARC_c_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_c_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-c',
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .ARC_c_gen_1e0de5 import ARC_c_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_gen_1e0de5.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:'
+                )
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ARC_c_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+)
+
+ARC_c_datasets = [
+    dict(
+        abbr='ARC-c',
+        type=ARCDataset,
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .ARC_c_ppl_a450bd import ARC_c_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_ppl_2ef631.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            opt: dict(
+                round=[
+                    dict(role='HUMAN', prompt=f'{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}'),
+                    dict(role='BOT', prompt=f'Answer: {opt}'),
+                ]
+            ) for opt in ['A', 'B', 'C', 'D']
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_c_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_c_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-c',
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_ppl_a450bd.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
+                ], ),
+            'B':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
+                ], ),
+            'C':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
+                ], ),
+            'D':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
+                ], ),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_c_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_c_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-c',
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_c/ARC_c_ppl_d52a21.py

@@ -0,0 +1,36 @@
+from mmengine.config import read_base
+# with read_base():
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_c_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_c_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A': 'Question: {question}\nAnswer: {textA}',
+            'B': 'Question: {question}\nAnswer: {textB}',
+            'C': 'Question: {question}\nAnswer: {textC}',
+            'D': 'Question: {question}\nAnswer: {textD}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_c_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_c_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-c',
+        path='opencompass/ai2_arc-dev',
+        name='ARC-Challenge',
+        reader_cfg=ARC_c_reader_cfg,
+        infer_cfg=ARC_c_infer_cfg,
+        eval_cfg=ARC_c_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .ARC_e_gen_1e0de5 import ARC_e_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_gen_1e0de5.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+ARC_e_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_e_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:'
+                )
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ARC_e_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+)
+
+ARC_e_datasets = [
+    dict(
+        abbr='ARC-e',
+        type=ARCDataset,
+        path='opencompass/ai2_arc-easy-dev',
+        name='ARC-Easy',
+        reader_cfg=ARC_e_reader_cfg,
+        infer_cfg=ARC_e_infer_cfg,
+        eval_cfg=ARC_e_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .ARC_e_ppl_a450bd import ARC_e_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_ppl_2ef631.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_e_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_e_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            opt: dict(
+                round=[
+                    dict(role='HUMAN', prompt=f'{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}'),
+                    dict(role='BOT', prompt=f'Answer: {opt}'),
+                ]
+            ) for opt in ['A', 'B', 'C', 'D']
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_e_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_e_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-e',
+        path='opencompass/ai2_arc-easy-dev',
+        name='ARC-Easy',
+        reader_cfg=ARC_e_reader_cfg,
+        infer_cfg=ARC_e_infer_cfg,
+        eval_cfg=ARC_e_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_ppl_a450bd.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_e_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_e_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
+                ], ),
+            'B':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
+                ], ),
+            'C':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
+                ], ),
+            'D':
+            dict(
+                round=[
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
+                ], ),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_e_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_e_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-e',
+        path='opencompass/ai2_arc-easy-dev',
+        name='ARC-Easy',
+        reader_cfg=ARC_e_reader_cfg,
+        infer_cfg=ARC_e_infer_cfg,
+        eval_cfg=ARC_e_eval_cfg)
+]

build/lib/opencompass/configs/datasets/ARC_e/ARC_e_ppl_d52a21.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_e_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_e_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A': 'Question: {question}\nAnswer: {textA}',
+            'B': 'Question: {question}\nAnswer: {textB}',
+            'C': 'Question: {question}\nAnswer: {textC}',
+            'D': 'Question: {question}\nAnswer: {textD}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_e_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_e_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-e',
+        path='opencompass/ai2_arc-easy-dev',
+        name='ARC-Easy',
+        reader_cfg=ARC_e_reader_cfg,
+        infer_cfg=ARC_e_infer_cfg,
+        eval_cfg=ARC_e_eval_cfg)
+]

build/lib/opencompass/configs/datasets/BeyondAIME/beyondaime_cascade_eval_gen_5e9f4f.py

@@ -0,0 +1,106 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import BeyondAIMEDataset
+from opencompass.evaluator import GenericLLMEvaluator, CascadeEvaluator, MATHVerifyEvaluator
+from opencompass.datasets import generic_llmjudge_postprocess
+
+
+beyondaime_reader_cfg = dict(input_columns=['question'], output_column='answer')
+
+
+GRADER_TEMPLATE = """
+    Please as a grading expert, judge whether the final answers given by the candidates below are consistent with the standard answers, that is, whether the candidates answered correctly. 
+
+    Here are some evaluation criteria:
+    1. Please refer to the given standard answer. You don't need to re-generate the answer to the question because the standard answer has been given. You only need to judge whether the candidate's answer is consistent with the standard answer according to the form of the question. Don't try to answer the original question. You can assume that the standard answer is definitely correct.
+    2. Because the candidate's answer may be different from the standard answer in the form of expression, before making a judgment, please understand the question and the standard answer first, and then judge whether the candidate's answer is correct, but be careful not to try to answer the original question.
+    3. Some answers may contain multiple items, such as multiple-choice questions, multiple-select questions, fill-in-the-blank questions, etc. As long as the answer is the same as the standard answer, it is enough. For multiple-select questions and multiple-blank fill-in-the-blank questions, the candidate needs to answer all the corresponding options or blanks correctly to be considered correct.
+    4. Some answers may be expressed in different ways, such as some answers may be a mathematical expression, some answers may be a textual description, as long as the meaning expressed is the same. And some formulas are expressed in different ways, but they are equivalent and correct.
+    5. If the prediction is given with \\boxed{}, please ignore the \\boxed{} and only judge whether the candidate's answer is consistent with the standard answer.
+
+    Please judge whether the following answers are consistent with the standard answer based on the above criteria. Grade the predicted answer of this new question as one of:
+    A: CORRECT 
+    B: INCORRECT
+    Just return the letters "A" or "B", with no text around it.
+
+    Here is your task. Simply reply with either CORRECT, INCORRECT. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer.
+
+
+    <Original Question Begin>: \n{question}\n<Original Question End>\n\n
+    <Gold Target Begin>: \n{answer}\n<Gold Target End>\n\n
+    <Predicted Answer Begin>: \n{prediction}\n<Predicted End>\n\n
+
+    Judging the correctness of candidates' answers:
+""".strip()
+
+
+
+beyondaime_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{question}\nRemember to put your final answer within \\boxed{}.',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+
+beyondaime_cascade_evaluator = dict(
+    type=CascadeEvaluator,
+    rule_evaluator=dict(
+        type=MATHVerifyEvaluator,
+    ),
+    llm_evaluator=dict(
+        type=GenericLLMEvaluator,
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                begin=[
+                    dict(
+                        role='SYSTEM',
+                        fallback_role='HUMAN',
+                        prompt="You are a helpful assistant who evaluates the correctness and quality of models' outputs.",
+                    )
+                ],
+                round=[
+                    dict(role='HUMAN', prompt=GRADER_TEMPLATE),
+                ],
+            ),
+        ),
+        dataset_cfg=dict(
+            type=BeyondAIMEDataset,
+            path='ByteDance-Seed/BeyondAIME',
+            reader_cfg=beyondaime_reader_cfg,
+        ),
+        judge_cfg=dict(),
+        dict_postprocessor=dict(type=generic_llmjudge_postprocess),
+    ),
+    parallel=False,
+)
+
+
+beyondaime_eval_cfg = dict(
+    evaluator=beyondaime_cascade_evaluator,
+)
+
+
+beyondaime_datasets = [
+    dict(
+        type=BeyondAIMEDataset,
+        abbr='beyondaime',
+        path='ByteDance-Seed/BeyondAIME',
+        reader_cfg=beyondaime_reader_cfg,
+        infer_cfg=beyondaime_infer_cfg,
+        eval_cfg=beyondaime_eval_cfg,
+    )
+]
+
+

build/lib/opencompass/configs/datasets/BeyondAIME/beyondaime_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .beyondaime_cascade_eval_gen_5e9f4f import beyondaime_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/CARDBiomedBench/CARDBiomedBench_llmjudge_gen_99a231.py

@@ -0,0 +1,101 @@
+from opencompass.datasets import CARDBiomedBenchDataset
+from opencompass.datasets import generic_llmjudge_postprocess
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.evaluator import GenericLLMEvaluator
+ZERO_SHOT_PROMPT = 'You are an expert in {expert}.\n{question}\n'
+
+GRADER_TEMPLATE = """
+    Please as a grading expert, judge whether the final answers given by the candidates below are consistent with the standard answers, that is, whether the candidates answered correctly. 
+    
+    Here are some evaluation criteria:
+    1. Please refer to the given standard answer. You don't need to re-generate the answer to the question because the standard answer has been given. You only need to judge whether the candidate's answer is consistent with the standard answer according to the form of the question. Don't try to answer the original question. You can assume that the standard answer is definitely correct.
+    2. Because the candidate's answer may be different from the standard answer in the form of expression, before making a judgment, please understand the question and the standard answer first, and then judge whether the candidate's answer is correct, but be careful not to try to answer the original question.
+    3. Some answers may contain multiple items, such as multiple-choice questions, multiple-select questions, fill-in-the-blank questions, etc. As long as the answer is the same as the standard answer, it is enough. For multiple-select questions and multiple-blank fill-in-the-blank questions, the candidate needs to answer all the corresponding options or blanks correctly to be considered correct.
+    4. Some answers may be expressed in different ways, such as some answers may be a mathematical expression, some answers may be a textual description, as long as the meaning expressed is the same. And some formulas are expressed in different ways, but they are equivalent and correct.
+
+    Please judge whether the following answers are consistent with the standard answer based on the above criteria. Grade the predicted answer of this new question as one of:
+    A: CORRECT 
+    B: INCORRECT
+    Just return the letters "A" or "B", with no text around it.
+
+    Here is your task. Simply reply with either CORRECT, INCORRECT. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer.
+
+    <Original Question Begin>: Q: You are an expert in {expert}.\n{question}\n<Original Question End>\n\n
+    <Gold Target Begin>: \n{answer}\n<Gold Target End>\n\n
+    <Predicted Answer Begin>: \n{prediction}\n<Predicted End>\n\n
+    Judging the correctness of candidates' answers:
+""".strip()
+
+
+# Reader configuration
+reader_cfg = dict(
+    input_columns=[
+        'question',
+        'answer',
+        'Bio_Category',
+        'SQL_Category',
+        'uuid',
+        'template uuid',
+        'expert',
+    ],
+    output_column='answer',
+)
+# Inference configuration
+infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    
+                    role='HUMAN',
+                    prompt=ZERO_SHOT_PROMPT, # prompt mode: zero-shot
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+# Evaluation configuration
+eval_cfg = dict(
+    evaluator=dict(
+        type=GenericLLMEvaluator,
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                begin=[
+                    dict(
+                        role='SYSTEM',
+                        fallback_role='HUMAN',
+                        prompt="You are a helpful assistant who evaluates the correctness and quality of models' outputs.",
+                    )
+                ],
+                round=[
+                    dict(role='HUMAN', prompt=GRADER_TEMPLATE),
+                ],
+            ),
+        ),
+        dataset_cfg=dict(
+            type=CARDBiomedBenchDataset,
+            path='NIH-CARD/CARDBiomedBench',
+            prompt_mode='zero-shot',
+            reader_cfg=reader_cfg,
+        ),
+        judge_cfg=dict(),
+        dict_postprocessor=dict(type=generic_llmjudge_postprocess),
+    ),
+)
+cardbiomedbench_dataset = dict(
+    type=CARDBiomedBenchDataset,
+    abbr='cardbiomedbench',
+    path='NIH-CARD/CARDBiomedBench',
+    prompt_mode='zero-shot',
+    reader_cfg=reader_cfg,
+    infer_cfg=infer_cfg,
+    eval_cfg=eval_cfg,
+)
+cardbiomedbench_datasets = [cardbiomedbench_dataset]

build/lib/opencompass/configs/datasets/CHARM/README.md

@@ -0,0 +1,164 @@
+# CHARM✨ Benchmarking Chinese Commonsense Reasoning of LLMs: From Chinese-Specifics to Reasoning-Memorization Correlations [ACL2024]
+[![arXiv](https://img.shields.io/badge/arXiv-2403.14112-b31b1b.svg)](https://arxiv.org/abs/2403.14112)
+[![license](https://img.shields.io/github/license/InternLM/opencompass.svg)](./LICENSE)
+<div align="center">
+
+📃[Paper](https://arxiv.org/abs/2403.14112)
+🏰[Project Page](https://opendatalab.github.io/CHARM/)
+🏆[Leaderboard](https://opendatalab.github.io/CHARM/leaderboard.html)
+✨[Findings](https://opendatalab.github.io/CHARM/findings.html)
+
+</div>
+
+<div align="center">
+    📖 <a href="./README_ZH.md">   中文</a> | <a href="./README.md">English</a>
+</div>
+
+## Dataset Description
+
+**CHARM** is the first benchmark for comprehensively and in-depth evaluating the commonsense reasoning ability of large language models (LLMs) in Chinese, which covers both globally known and Chinese-specific commonsense. In addition, the CHARM can evaluate the LLMs' memorization-independent reasoning abilities and analyze the typical errors.
+
+## Comparison of commonsense reasoning benchmarks
+<html lang="en">
+        <table align="center">
+            <thead class="fixed-header">
+                <tr>
+                    <th>Benchmarks</th>
+                    <th>CN-Lang</th>
+                    <th>CSR</th>
+                    <th>CN-specifics</th>
+                    <th>Dual-Domain</th>
+                    <th>Rea-Mem</th>
+                </tr>
+            </thead>
+            <tr>
+                <td>Most benchmarks in <a href="https://arxiv.org/abs/2302.04752"> davis2023benchmarks</a></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/1809.05053"> XNLI</a>, <a
+                        href="https://arxiv.org/abs/2005.00333">XCOPA</a>,<a
+                        href="https://arxiv.org/abs/2112.10668">XStoryCloze</a></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/2007.08124">LogiQA</a>, <a
+                        href="https://arxiv.org/abs/2004.05986">CLUE</a>, <a
+                        href="https://arxiv.org/abs/2306.09212">CMMLU</a></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/2312.12853">CORECODE</a> </td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><strong><a href="https://arxiv.org/abs/2403.14112">CHARM (ours)</a> </strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+            </tr>
+        </table>
+
+"CN-Lang" indicates the benchmark is presented in Chinese language. "CSR" means the benchmark is designed to focus on <strong>C</strong>ommon<strong>S</strong>ense <strong>R</strong>easoning. "CN-specific" indicates the benchmark includes elements that are unique to Chinese culture, language, regional characteristics, history, etc. "Dual-Domain" indicates the benchmark encompasses both Chinese-specific and global domain tasks, with questions presented in the similar style and format. "Rea-Mem" indicates the benchmark includes closely-interconnected <strong>rea</strong>soning and <strong>mem</strong>orization tasks.
+
+
+## 🛠️ How to Use
+Below are the steps for quickly downloading CHARM and using OpenCompass for evaluation.
+
+### 1. Download CHARM
+```bash
+git clone https://github.com/opendatalab/CHARM ${path_to_CHARM_repo}
+
+cd ${path_to_opencompass}
+mkdir data
+ln -snf ${path_to_CHARM_repo}/data/CHARM ./data/CHARM
+```
+### 2. Run Inference and Evaluation
+```bash
+cd ${path_to_opencompass}
+
+# modify config file `examples/eval_charm_rea.py`: uncomment or add models you want to evaluate
+python run.py examples/eval_charm_rea.py -r --dump-eval-details
+
+# modify config file `examples/eval_charm_mem.py`: uncomment or add models you want to evaluate
+python run.py examples/eval_charm_mem.py -r --dump-eval-details
+```
+The inference and evaluation results would be in `${path_to_opencompass}/outputs`, like this:
+```bash
+outputs
+├── CHARM_mem
+│   └── chat
+│       └── 20240605_151442
+│           ├── predictions
+│           │   ├── internlm2-chat-1.8b-turbomind
+│           │   ├── llama-3-8b-instruct-lmdeploy
+│           │   └── qwen1.5-1.8b-chat-hf
+│           ├── results
+│           │   ├── internlm2-chat-1.8b-turbomind_judged-by--GPT-3.5-turbo-0125
+│           │   ├── llama-3-8b-instruct-lmdeploy_judged-by--GPT-3.5-turbo-0125
+│           │   └── qwen1.5-1.8b-chat-hf_judged-by--GPT-3.5-turbo-0125
+│           └── summary
+│               └── 20240605_205020 # MEMORY_SUMMARY_DIR
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Anachronisms_Judgment
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Movie_and_Music_Recommendation
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Sport_Understanding
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Time_Understanding
+│                   └── judged-by--GPT-3.5-turbo-0125.csv # MEMORY_SUMMARY_CSV
+└── CHARM_rea
+    └── chat
+        └── 20240605_152359
+            ├── predictions
+            │   ├── internlm2-chat-1.8b-turbomind
+            │   ├── llama-3-8b-instruct-lmdeploy
+            │   └── qwen1.5-1.8b-chat-hf
+            ├── results # REASON_RESULTS_DIR
+            │   ├── internlm2-chat-1.8b-turbomind
+            │   ├── llama-3-8b-instruct-lmdeploy
+            │   └── qwen1.5-1.8b-chat-hf
+            └── summary
+                ├── summary_20240605_205328.csv # REASON_SUMMARY_CSV
+                └── summary_20240605_205328.txt
+```
+### 3. Generate Analysis Results
+```bash
+cd ${path_to_CHARM_repo}
+
+# generate Table5, Table6, Table9 and Table10 in https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/summarize_reasoning.py ${REASON_SUMMARY_CSV}
+
+# generate Figure3 and Figure9 in https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/summarize_mem_rea.py ${REASON_SUMMARY_CSV} ${MEMORY_SUMMARY_CSV}
+
+# generate Table7, Table12, Table13 and Figure11 in https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/analyze_mem_indep_rea.py data/CHARM ${REASON_RESULTS_DIR} ${MEMORY_SUMMARY_DIR} ${MEMORY_SUMMARY_CSV}
+```
+
+## 🖊️ Citation
+```bibtex
+@misc{sun2024benchmarking,
+      title={Benchmarking Chinese Commonsense Reasoning of LLMs: From Chinese-Specifics to Reasoning-Memorization Correlations},
+      author={Jiaxing Sun and Weiquan Huang and Jiang Wu and Chenya Gu and Wei Li and Songyang Zhang and Hang Yan and Conghui He},
+      year={2024},
+      eprint={2403.14112},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

build/lib/opencompass/configs/datasets/CHARM/README_ZH.md

@@ -0,0 +1,162 @@
+# CHARM✨ Benchmarking Chinese Commonsense Reasoning of LLMs: From Chinese-Specifics to Reasoning-Memorization Correlations [ACL2024]
+[![arXiv](https://img.shields.io/badge/arXiv-2403.14112-b31b1b.svg)](https://arxiv.org/abs/2403.14112)
+[![license](https://img.shields.io/github/license/InternLM/opencompass.svg)](./LICENSE)
+<div align="center">
+
+📃[Paper](https://arxiv.org/abs/2403.14112)
+🏰[Project Page](https://opendatalab.github.io/CHARM/)
+🏆[Leaderboard](https://opendatalab.github.io/CHARM/leaderboard.html)
+✨[Findings](https://opendatalab.github.io/CHARM/findings.html)
+</div>
+
+<div align="center">
+    📖 <a href="./README_ZH.md">   中文</a> | <a href="./README.md">English</a>
+</div>
+
+## 数据集介绍
+
+**CHARM** 是首个全面深入评估大型语言模型（LLMs）在中文常识推理能力的基准测试，它覆盖了国际普遍认知的常识以及独特的中国文化常识。此外，CHARM 还可以评估 LLMs 独立于记忆的推理能力，并分析其典型错误。
+
+
+## 与其他常识推理评测基准的比较
+<html lang="en">
+        <table align="center">
+            <thead class="fixed-header">
+                <tr>
+                    <th>基准</th>
+                    <th>汉语</th>
+                    <th>常识推理</th>
+                    <th>中国特有知识</th>
+                    <th>中国和世界知识域</th>
+                    <th>推理和记忆的关系</th>
+                </tr>
+            </thead>
+            <tr>
+                <td><a href="https://arxiv.org/abs/2302.04752"> davis2023benchmarks</a> 中提到的基准</td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/1809.05053"> XNLI</a>, <a
+                        href="https://arxiv.org/abs/2005.00333">XCOPA</a>,<a
+                        href="https://arxiv.org/abs/2112.10668">XStoryCloze</a></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/2007.08124">LogiQA</a>,<a
+                        href="https://arxiv.org/abs/2004.05986">CLUE</a>, <a
+                        href="https://arxiv.org/abs/2306.09212">CMMLU</a></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><a href="https://arxiv.org/abs/2312.12853">CORECODE</a> </td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+                <td><strong><span style="color: red;">&#x2718;</span></strong></td>
+            </tr>
+            <tr>
+                <td><strong><a href="https://arxiv.org/abs/2403.14112">CHARM (ours)</a> </strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+                <td><strong><span style="color: green;">&#x2714;</span></strong></td>
+            </tr>
+        </table>
+
+
+## 🛠️ 如何使用
+以下是快速下载 CHARM 并在 OpenCompass 上进行评估的步骤。
+
+### 1. 下载 CHARM
+```bash
+git clone https://github.com/opendatalab/CHARM ${path_to_CHARM_repo}
+
+cd ${path_to_opencompass}
+mkdir data
+ln -snf ${path_to_CHARM_repo}/data/CHARM ./data/CHARM
+```
+### 2. 推理和评测
+```bash
+cd ${path_to_opencompass}
+
+# 修改配置文件`examples/eval_charm_rea.py`: 将现有的模型取消注释，或者添加你想评测的模型
+python run.py examples/eval_charm_rea.py -r --dump-eval-details
+
+# 修改配置文件`examples/eval_charm_mem.py`: 将现有的模型取消注释，或者添加你想评测的模型
+python run.py examples/eval_charm_mem.py -r --dump-eval-details
+```
+推理和评测的结果位于路径`${path_to_opencompass}/outputs`, 如下所示:
+```bash
+outputs
+├── CHARM_mem
+│   └── chat
+│       └── 20240605_151442
+│           ├── predictions
+│           │   ├── internlm2-chat-1.8b-turbomind
+│           │   ├── llama-3-8b-instruct-lmdeploy
+│           │   └── qwen1.5-1.8b-chat-hf
+│           ├── results
+│           │   ├── internlm2-chat-1.8b-turbomind_judged-by--GPT-3.5-turbo-0125
+│           │   ├── llama-3-8b-instruct-lmdeploy_judged-by--GPT-3.5-turbo-0125
+│           │   └── qwen1.5-1.8b-chat-hf_judged-by--GPT-3.5-turbo-0125
+│           └── summary
+│               └── 20240605_205020 # MEMORY_SUMMARY_DIR
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Anachronisms_Judgment
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Movie_and_Music_Recommendation
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Sport_Understanding
+│                   ├── judged-by--GPT-3.5-turbo-0125-charm-memory-Chinese_Time_Understanding
+│                   └── judged-by--GPT-3.5-turbo-0125.csv # MEMORY_SUMMARY_CSV
+└── CHARM_rea
+    └── chat
+        └── 20240605_152359
+            ├── predictions
+            │   ├── internlm2-chat-1.8b-turbomind
+            │   ├── llama-3-8b-instruct-lmdeploy
+            │   └── qwen1.5-1.8b-chat-hf
+            ├── results # REASON_RESULTS_DIR
+            │   ├── internlm2-chat-1.8b-turbomind
+            │   ├── llama-3-8b-instruct-lmdeploy
+            │   └── qwen1.5-1.8b-chat-hf
+            └── summary
+                ├── summary_20240605_205328.csv # REASON_SUMMARY_CSV
+                └── summary_20240605_205328.txt
+```
+### 3. 生成分析结果
+```bash
+cd ${path_to_CHARM_repo}
+
+# 生成论文中的Table5, Table6, Table9 and Table10，详见https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/summarize_reasoning.py ${REASON_SUMMARY_CSV}
+
+# 生成论文中的Figure3 and Figure9，详见https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/summarize_mem_rea.py ${REASON_SUMMARY_CSV} ${MEMORY_SUMMARY_CSV}
+
+# 生成论文中的Table7, Table12, Table13 and Figure11，详见https://arxiv.org/abs/2403.14112
+PYTHONPATH=. python tools/analyze_mem_indep_rea.py data/CHARM ${REASON_RESULTS_DIR} ${MEMORY_SUMMARY_DIR} ${MEMORY_SUMMARY_CSV}
+```
+
+## 🖊️ 引用
+```bibtex
+@misc{sun2024benchmarking,
+      title={Benchmarking Chinese Commonsense Reasoning of LLMs: From Chinese-Specifics to Reasoning-Memorization Correlations},
+      author={Jiaxing Sun and Weiquan Huang and Jiang Wu and Chenya Gu and Wei Li and Songyang Zhang and Hang Yan and Conghui He},
+      year={2024},
+      eprint={2403.14112},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

build/lib/opencompass/configs/datasets/CHARM/charm_memory_gen_bbbd53.py

@@ -0,0 +1,63 @@
+import os
+from mmengine.config import read_base
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CharmDataset, CharmMemoryEvaluator, LMEvaluator
+
+with read_base():
+    from .charm_memory_settings import charm_memory_tasks, judge_system_prompts, dataset_path
+
+charm_memory_datasets = []
+
+for _task in charm_memory_tasks:
+
+    charm_memory_reader_cfg = dict(input_columns=['input'],
+                                   output_column='target')
+
+    charm_memory_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(role='HUMAN', prompt='请尽可能简短地回答下述问题。\n问题：{input}\n答：')
+            ]),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=512),
+    )
+
+    if _task == 'Chinese_Movie_and_Music_Recommendation':
+        charm_memory_eval_cfg = dict(
+            evaluator=dict(type=CharmMemoryEvaluator),
+            pred_role='BOT',
+        )
+    else:
+        judge_system_prompt = judge_system_prompts[_task]
+        charm_memory_eval_cfg = dict(
+            evaluator=dict(
+                type=LMEvaluator,
+                prompt_template=dict(
+                    type=PromptTemplate,
+                    template=dict(round=[
+                        dict(
+                            role='HUMAN',
+                            prompt=judge_system_prompt +
+                            "\n\n[Question]\n{input}\n[The Start of Reference Answer]\n{target}\n[The End of Reference Answer]\n\n[The Start of Assistant's Answer]\n{prediction}\n[The End of Assistant's Answer]"  # noqa
+                        ),
+                    ]),
+                ),
+            ),
+            pred_role='BOT',
+        )
+
+    charm_memory_datasets.append(
+        dict(
+            type=CharmDataset,
+            path=dataset_path,
+            name=_task,
+            abbr='charm-memory-' + _task,
+            reader_cfg=charm_memory_reader_cfg,
+            infer_cfg=charm_memory_infer_cfg.copy(),
+            eval_cfg=charm_memory_eval_cfg.copy(),
+        ))

build/lib/opencompass/configs/datasets/CHARM/charm_memory_settings.py

@@ -0,0 +1,31 @@
+import os
+
+charm_memory_tasks = [
+    'Chinese_Anachronisms_Judgment',
+    'Chinese_Movie_and_Music_Recommendation',
+    'Chinese_Sport_Understanding',
+    'Chinese_Time_Understanding',
+]
+
+dataset_path = 'data/CHARM/memorization'
+
+system_prompt_template = """Please act as an impartial judge, comparing the responses of the AI assistants to the reference answer and determining if the answers are correct.
+You will receive the reference answer provided by a human and the responses of the AI assistants.
+Your task is to judge whether the AI assistant's answers is correct.
+{task_specific_prompt}
+After providing your explanation, strictly output your final judgment in the following format: “[正确]” if the AI assistant's response is correct, “[错误]” if the AI assistant's response is incorrect.
+"""
+
+task_specific_prompts = {
+    'Chinese_Anachronisms_Judgment':
+    "If the provided reference answer is a list, the model's prediction is considered correct if it matches any item in the list.",
+    'Chinese_Time_Understanding':
+    "When evaluating the AI assistant's response regarding Chinese solar terms, as long as the AI assistant's response falls within the time frame provided in the reference answer, consider it correct.",
+    'Chinese_Sport_Understanding':
+    "If the provided reference answer is a list, the model's prediction is considered correct if it matches any item in the list."
+}
+
+judge_system_prompts = {
+    k: system_prompt_template.format(task_specific_prompt=v)
+    for k, v in task_specific_prompts.items()
+}

build/lib/opencompass/configs/datasets/CHARM/charm_reason_cot_only_gen_f7b7d3.py

@@ -0,0 +1,50 @@
+import os
+from mmengine.config import read_base
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CharmDataset, charm_reason_postprocess, CharmReasonEvaluator
+
+with read_base():
+    from .charm_reason_settings import charm_tasks, settings
+
+settings = [s for s in settings if s[0] in ['ZH-CoT', 'EN-CoT']]
+
+charm_reason_datasets = []
+
+for _cot, _cot_prefix, dataset_path, fewshot_example_path, prompt_template in settings:
+    for _task in charm_tasks:
+        _fewshot_example_file = os.path.join(fewshot_example_path, f'{_task}_{_cot}.txt')
+        with open(_fewshot_example_file, 'r') as f:
+            _hint = f.read()
+
+        charm_reason_reader_cfg = dict(input_columns=['input'], output_column='target')
+
+        charm_reason_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[dict(role='HUMAN', prompt=prompt_template.format(_hint=_hint) + _cot_prefix)]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=512),
+        )
+
+        charm_reason_eval_cfg = dict(
+            evaluator=dict(type=CharmReasonEvaluator),
+            pred_role='BOT',
+            pred_postprocessor=dict(type=charm_reason_postprocess),
+            dataset_postprocessor=dict(type=charm_reason_postprocess),
+        )
+
+        charm_reason_datasets.append(
+            dict(
+                type=CharmDataset,
+                path=dataset_path,
+                name=_task,
+                abbr='charm-reason-' + _task + '_' + _cot,
+                reader_cfg=charm_reason_reader_cfg,
+                infer_cfg=charm_reason_infer_cfg.copy(),
+                eval_cfg=charm_reason_eval_cfg.copy(),
+            )
+        )

build/lib/opencompass/configs/datasets/CHARM/charm_reason_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .charm_reason_gen_f8fca2 import charm_reason_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/CHARM/charm_reason_gen_f8fca2.py

@@ -0,0 +1,49 @@
+import os
+from mmengine.config import read_base
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CharmDataset, charm_reason_postprocess, CharmReasonEvaluator
+
+with read_base():
+    from .charm_reason_settings import charm_tasks, settings
+
+
+charm_reason_datasets = []
+
+for _cot, _cot_prefix, dataset_path, fewshot_example_path, prompt_template in settings:
+    for _task in charm_tasks:
+        _fewshot_example_file = os.path.join(fewshot_example_path, f'{_task}_{_cot}.txt')
+        with open(_fewshot_example_file, 'r') as f:
+            _hint = f.read()
+
+        charm_reason_reader_cfg = dict(input_columns=['input'], output_column='target')
+
+        charm_reason_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[dict(role='HUMAN', prompt=prompt_template.format(_hint=_hint) + _cot_prefix)]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=512),
+        )
+
+        charm_reason_eval_cfg = dict(
+            evaluator=dict(type=CharmReasonEvaluator),
+            pred_role='BOT',
+            pred_postprocessor=dict(type=charm_reason_postprocess),
+            dataset_postprocessor=dict(type=charm_reason_postprocess),
+        )
+
+        charm_reason_datasets.append(
+            dict(
+                type=CharmDataset,
+                path=dataset_path,
+                name=_task,
+                abbr='charm-reason-' + _task + '_' + _cot,
+                reader_cfg=charm_reason_reader_cfg,
+                infer_cfg=charm_reason_infer_cfg.copy(),
+                eval_cfg=charm_reason_eval_cfg.copy(),
+            )
+        )

build/lib/opencompass/configs/datasets/CHARM/charm_reason_ppl_3da4de.py

@@ -0,0 +1,57 @@
+import os
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.datasets import CharmDataset
+from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator
+
+charm_tasks = [
+    ['Chinese_Anachronisms_Judgment', 'AB'],
+    ['Chinese_Movie_and_Music_Recommendation', 'ABCD'],
+    ['Chinese_Natural_Language_Inference', 'ABC'],
+    ['Chinese_Reading_Comprehension', 'ABCD'],
+    ['Chinese_Sequence_Understanding', 'ABCD'],
+    ['Chinese_Sport_Understanding', 'AB'],
+    ['Chinese_Time_Understanding', 'ABCD'],
+    ['Global_Anachronisms_Judgment', 'AB'],
+    ['Global_Movie_and_Music_Recommendation', 'ABCD'],
+    ['Global_Natural_Language_Inference', 'ABC'],
+    ['Global_Reading_Comprehension', 'ABCD'],
+    ['Global_Sequence_Understanding', 'ABCD'],
+    ['Global_Sport_Understanding', 'AB'],
+    ['Global_Time_Understanding', 'ABCDEF'],
+]
+
+charm_reason_datasets = []
+for task_name, options in charm_tasks:
+
+    with open(os.path.join(os.path.dirname(__file__), 'few-shot-examples', f'{task_name}_Direct.txt'), 'r') as f:
+        few_shot_example = f.read()
+
+    charm_reason_reader_cfg = dict(input_columns=['input'], output_column='target')
+
+    charm_reason_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                f'({opt})': f'{few_shot_example}\n{{input}}\nA: {opt}' for opt in options
+            },
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=PPLInferencer),
+    )
+
+    charm_reason_eval_cfg = dict(evaluator=dict(type=AccwithDetailsEvaluator))
+
+    charm_reason_datasets.append(
+        dict(
+            type=CharmDataset,
+            abbr=f'charm-reason-{task_name}_Direct',
+            path=f'data/CHARM/reasoning',
+            name=task_name,
+            reader_cfg=charm_reason_reader_cfg,
+            infer_cfg=charm_reason_infer_cfg,
+            eval_cfg=charm_reason_eval_cfg,
+        )
+    )

build/lib/opencompass/configs/datasets/CHARM/charm_reason_settings.py

@@ -0,0 +1,36 @@
+import os
+
+charm_tasks = [
+    'Chinese_Anachronisms_Judgment',
+    'Chinese_Movie_and_Music_Recommendation',
+    'Chinese_Natural_Language_Inference',
+    'Chinese_Reading_Comprehension',
+    'Chinese_Sequence_Understanding',
+    'Chinese_Sport_Understanding',
+    'Chinese_Time_Understanding',
+    'Global_Anachronisms_Judgment',
+    'Global_Movie_and_Music_Recommendation',
+    'Global_Natural_Language_Inference',
+    'Global_Reading_Comprehension',
+    'Global_Sequence_Understanding',
+    'Global_Sport_Understanding',
+    'Global_Time_Understanding',
+]
+
+XLT_template = 'Follow the given examples and answer the question.\n{_hint}\n\n I want you to act as an commonsense reasoning expert for Chinese. \n Request: {{input}}\n'
+Translate_EN_template = 'Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: '
+Other_template = '请按照给定的例子回答问题。\n{_hint}\n\nQ：{{input}}\nA：'
+
+data_dir = 'data/CHARM'
+dataset_path_ZH = f'{data_dir}/reasoning'
+dataset_path_TransEn = f'{data_dir}/reasoning_Translate-EN'
+fewshot_example_path_ZH = os.path.join(os.path.dirname(__file__), 'few-shot-examples')
+fewshot_example_path_TransEn = os.path.join(os.path.dirname(__file__), 'few-shot-examples_Translate-EN')
+
+settings = [
+    ('Direct', '', dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('ZH-CoT', '让我们一步一步来思考。', dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('EN-CoT', "Let's think step by step.", dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('XLT', """You should retell the request in English.\nYou should do the answer step by step to choose the right answer.\nYou should step-by-step answer the request.\nYou should tell me the answer in this format 'So the answer is'.""", dataset_path_ZH, fewshot_example_path_ZH, XLT_template),
+    ('Translate-EN', "Let's think step by step.", dataset_path_TransEn, fewshot_example_path_TransEn, Translate_EN_template),
+]

build/lib/opencompass/configs/datasets/CIBench/CIBench_generation_gen_8ab0dc.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every'),
+)
+
+libs = ['matplotlib', 'opencv', 'pandas', 'pytorch', 'scipy', 'seaborn']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_generation/{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_generation/{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

build/lib/opencompass/configs/datasets/CIBench/CIBench_generation_oracle_gen_c4a7c1.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every_with_gt'),
+)
+
+libs = ['matplotlib', 'opencv', 'pandas', 'pytorch', 'scipy', 'seaborn']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_generation_oracle/{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_generation/{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

build/lib/opencompass/configs/datasets/CIBench/CIBench_template_gen_e6b12a.py

@@ -0,0 +1,39 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every'),
+)
+
+# no tensorboard
+libs = ['/lightgbm', '/matplotlib', '/nltk', '/opencv', '/pandas', '/pytorch',
+    '/scipy', '/seaborn', '/sklearn', '/tensorflow',
+    '_chinese/lightgbm', '_chinese/matplotlib', '_chinese/nltk',
+    '_chinese/opencv', '_chinese/pandas', '_chinese/pytorch',
+    '_chinese/scipy', '_chinese/seaborn', '_chinese/sklearn', '_chinese/tensorflow']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_template{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_template{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

build/lib/opencompass/configs/datasets/CIBench/CIBench_template_oracle_gen_fecda1.py

@@ -0,0 +1,39 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every_with_gt'),
+)
+
+# no tensorboard
+libs = ['/lightgbm', '/matplotlib', '/nltk', '/opencv', '/pandas', '/pytorch',
+    '/scipy', '/seaborn', '/sklearn', '/tensorflow',
+    '_chinese/lightgbm', '_chinese/matplotlib', '_chinese/nltk',
+    '_chinese/opencv', '_chinese/pandas', '_chinese/pytorch',
+    '_chinese/scipy', '_chinese/seaborn', '_chinese/sklearn', '_chinese/tensorflow']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_template_oracle{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_template{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

build/lib/opencompass/configs/datasets/CLUE_C3/CLUE_C3_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_C3_gen_8c358f import C3_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/CLUE_C3/CLUE_C3_gen_8c358f.py

@@ -0,0 +1,51 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question',
+        'content',
+        'choice0',
+        'choice1',
+        'choice2',
+        'choice3',
+        'choices',
+    ],
+    output_column='label',
+)
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{content}\n问：{question}\nA. {choice0}\nB. {choice1}\nC. {choice2}\nD. {choice3}\n请从“A”，“B”，“C”，“D”中进行选择。\n答：',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+C3_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+C3_datasets = [
+    dict(
+        abbr='C3',
+        type=C3Dataset_V2,
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_C3_ppl_e24a31 import C3_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl_56b537.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3',
+        'choices'
+    ],
+    output_column='label')
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '文章：{content}\n问题：{question}\n答案：{choice0}',
+            1: '文章：{content}\n问题：{question}\n答案：{choice1}',
+            2: '文章：{content}\n问题：{question}\n答案：{choice2}',
+            3: '文章：{content}\n问题：{question}\n答案：{choice3}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+C3_datasets = [
+    dict(
+        type=C3Dataset,
+        abbr='C3',
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg)
+]

build/lib/opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl_e24a31.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3',
+        'choices'
+    ],
+    output_column='label')
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            i: dict(round=[
+                dict(role='HUMAN', prompt='文章：{content}\n问题：{question}'),
+                dict(role='BOT', prompt=f'答案：{{choice{i}}}')
+            ])
+            for i in range(4)
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+C3_datasets = [
+    dict(
+        type=C3Dataset,
+        abbr='C3',
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg)
+]

build/lib/opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_CMRC_gen_1bd3c8 import CMRC_datasets  # noqa: F401, F403