Add files using upload-large-folder tool

build/lib/opencompass/configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_passage_retrieval_en_gen_734db5 import LongBench_passage_retrieval_en_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen_734db5.py

@@ -0,0 +1,46 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_enDataset,
+)
+
+LongBench_passage_retrieval_en_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_passage_retrieval_en_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32),
+)
+
+LongBench_passage_retrieval_en_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRetrievalEvaluator), pred_role='BOT'
+)
+
+LongBench_passage_retrieval_en_datasets = [
+    dict(
+        type=LongBenchpassage_retrieval_enDataset,
+        abbr='LongBench_passage_retrieval_en',
+        path='opencompass/Longbench',
+        name='passage_retrieval_en',
+        reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
+        infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
+        eval_cfg=LongBench_passage_retrieval_en_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_passage_retrieval_zh_gen_01cca2 import LongBench_passage_retrieval_zh_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen_01cca2.py

@@ -0,0 +1,46 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_zhDataset,
+)
+
+LongBench_passage_retrieval_zh_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_passage_retrieval_zh_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32),
+)
+
+LongBench_passage_retrieval_zh_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'), pred_role='BOT'
+)
+
+LongBench_passage_retrieval_zh_datasets = [
+    dict(
+        type=LongBenchpassage_retrieval_zhDataset,
+        abbr='LongBench_passage_retrieval_zh',
+        path='opencompass/Longbench',
+        name='passage_retrieval_zh',
+        reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
+        infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
+        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchqasper/longbench_qasper_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_qasper_gen_6b3efc import LongBench_qasper_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchqasper/longbench_qasper_gen_6b3efc.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchF1Evaluator, LongBenchqasperDataset
+
+LongBench_qasper_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_qasper_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32),
+)
+
+LongBench_qasper_eval_cfg = dict(
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
+)
+
+LongBench_qasper_datasets = [
+    dict(
+        type=LongBenchqasperDataset,
+        abbr='LongBench_qasper',
+        path='opencompass/Longbench',
+        name='qasper',
+        reader_cfg=LongBench_qasper_reader_cfg,
+        infer_cfg=LongBench_qasper_infer_cfg,
+        eval_cfg=LongBench_qasper_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_qmsum_gen_d33331 import LongBench_qmsum_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen_d33331.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRougeEvaluator, LongBenchqmsumDataset
+
+LongBench_qmsum_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_qmsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+LongBench_qmsum_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
+)
+
+LongBench_qmsum_datasets = [
+    dict(
+        type=LongBenchqmsumDataset,
+        abbr='LongBench_qmsum',
+        path='opencompass/Longbench',
+        name='qmsum',
+        reader_cfg=LongBench_qmsum_reader_cfg,
+        infer_cfg=LongBench_qmsum_infer_cfg,
+        eval_cfg=LongBench_qmsum_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_repobench_gen_6df953 import LongBench_repobench_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen_6df953.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchCodeSimEvaluator, LongBenchrepobenchDataset
+
+LongBench_repobench_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_repobench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please complete the code given below. \n{context}{input}Next line of code:\n',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LongBench_repobench_eval_cfg = dict(
+    evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
+)
+
+LongBench_repobench_datasets = [
+    dict(
+        type=LongBenchrepobenchDataset,
+        abbr='LongBench_repobench-p',
+        path='opencompass/Longbench',
+        name='repobench-p',
+        reader_cfg=LongBench_repobench_reader_cfg,
+        infer_cfg=LongBench_repobench_infer_cfg,
+        eval_cfg=LongBench_repobench_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchsamsum/longbench_samsum_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_samsum_gen_f4416d import LongBench_samsum_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchsamsum/longbench_samsum_gen_f4416d.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LongBenchRougeEvaluator,
+    LongBenchsamsumDataset,
+    samsum_postprocess,
+)
+
+LongBench_samsum_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_samsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=128),
+)
+
+LongBench_samsum_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRougeEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=samsum_postprocess),
+)
+
+LongBench_samsum_datasets = [
+    dict(
+        type=LongBenchsamsumDataset,
+        abbr='LongBench_samsum',
+        path='opencompass/Longbench',
+        name='samsum',
+        reader_cfg=LongBench_samsum_reader_cfg,
+        infer_cfg=LongBench_samsum_infer_cfg,
+        eval_cfg=LongBench_samsum_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchtrec/longbench_trec_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_trec_gen_824187 import LongBench_trec_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchtrec/longbench_trec_gen_824187.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LongBenchClassificationEvaluator,
+    LongBenchtrecDataset,
+    trec_postprocess,
+)
+
+LongBench_trec_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='all_labels',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_trec_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LongBench_trec_eval_cfg = dict(
+    evaluator=dict(type=LongBenchClassificationEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=trec_postprocess),
+)
+
+LongBench_trec_datasets = [
+    dict(
+        type=LongBenchtrecDataset,
+        abbr='LongBench_trec',
+        path='opencompass/Longbench',
+        name='trec',
+        reader_cfg=LongBench_trec_reader_cfg,
+        infer_cfg=LongBench_trec_infer_cfg,
+        eval_cfg=LongBench_trec_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_triviaqa_gen_d30cb9 import LongBench_triviaqa_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen_d30cb9.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LongBenchF1Evaluator,
+    LongBenchtriviaqaDataset,
+    triviaqa_postprocess,
+)
+
+LongBench_triviaqa_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_triviaqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=32),
+)
+
+LongBench_triviaqa_eval_cfg = dict(
+    evaluator=dict(type=LongBenchF1Evaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=triviaqa_postprocess),
+)
+
+LongBench_triviaqa_datasets = [
+    dict(
+        type=LongBenchtriviaqaDataset,
+        abbr='LongBench_triviaqa',
+        path='opencompass/Longbench',
+        name='triviaqa',
+        reader_cfg=LongBench_triviaqa_reader_cfg,
+        infer_cfg=LongBench_triviaqa_infer_cfg,
+        eval_cfg=LongBench_triviaqa_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .longbench_vcsum_gen_f7a8ac import LongBench_vcsum_datasets  # noqa: F401, F403

build/lib/opencompass/configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen_f7a8ac.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LongBenchRougeEvaluator, LongBenchvcsumDataset
+
+LongBench_vcsum_reader_cfg = dict(
+    input_columns=['context'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LongBench_vcsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+LongBench_vcsum_eval_cfg = dict(
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
+)
+
+LongBench_vcsum_datasets = [
+    dict(
+        type=LongBenchvcsumDataset,
+        abbr='LongBench_vcsum',
+        path='opencompass/Longbench',
+        name='vcsum',
+        reader_cfg=LongBench_vcsum_reader_cfg,
+        infer_cfg=LongBench_vcsum_infer_cfg,
+        eval_cfg=LongBench_vcsum_eval_cfg,
+    )
+]

build/lib/opencompass/configs/datasets/lveval/lvevalcmrc_mixup/lveval_cmrc_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_cmrc_mixup_gen_465823 import (
+        LVEval_cmrc_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalcmrc_mixup/lveval_cmrc_mixup_gen_465823.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTF1Evaluator, LVEvalcmrcDataset
+
+LVEval_cmrc_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_cmrc_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='请根据下面给定的文章回答问题，问题和答案只与其中一篇文章有关。\n\n文章：{context}\n\n现在请基于上述文章回答下面的问题，问题和答案只与其中一篇文章有关。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_cmrc_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='zh'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_cmrc_mixup_datasets = [
+    dict(
+        type=LVEvalcmrcDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_cmrc_mixup_reader_cfg,
+        infer_cfg=LVEval_cmrc_mixup_infer_cfg,
+        eval_cfg=LVEval_cmrc_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('cmrc_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevaldureader_mixup/lveval_dureader_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_dureader_mixup_gen_465823 import (
+        LVEval_dureader_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevaldureader_mixup/lveval_dureader_mixup_gen_465823.py

@@ -0,0 +1,55 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTRougeEvaluator, LVEvaldureaderDataset
+
+LVEval_dureader_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_dureader_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='请根据下面给定的文章回答问题，问题和答案只与其中一篇文章有关。\n\n文章：{context}\n\n现在请基于上述文章回答下面的问题，问题和答案只与其中一篇文章有关。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_dureader_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTRougeEvaluator, language='zh'),
+    pred_role='BOT',
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_dureader_mixup_datasets = [
+    dict(
+        type=LVEvaldureaderDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_dureader_mixup_reader_cfg,
+        infer_cfg=LVEval_dureader_mixup_infer_cfg,
+        eval_cfg=LVEval_dureader_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('dureader_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalfactrecall_en/lveval_factrecall_en_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_factrecall_en_gen_9a836f import (
+        LVEval_factrecall_en_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalfactrecall_en/lveval_factrecall_en_gen_9a836f.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalF1Evaluator, LVEvalfactrecallenDataset
+
+LVEval_factrecall_en_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_factrecall_en_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please answer the following questions based on the given article.\n\nArticle: {context}\n\nPlease answer the following questions based on the above article.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=16),
+)
+
+LVEval_factrecall_en_eval_cfg = dict(
+    evaluator=dict(type=LVEvalF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_factrecall_en_datasets = [
+    dict(
+        type=LVEvalfactrecallenDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_factrecall_en_reader_cfg,
+        infer_cfg=LVEval_factrecall_en_infer_cfg,
+        eval_cfg=LVEval_factrecall_en_eval_cfg,
+    )
+    for name_len in get_dataset_names('factrecall_en', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalfactrecall_zh/lveval_factrecall_zh_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_factrecall_zh_gen_dbee70 import (
+        LVEval_factrecall_zh_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalfactrecall_zh/lveval_factrecall_zh_gen_dbee70.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalF1Evaluator, LVEvalfactrecallzhDataset
+
+LVEval_factrecall_zh_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_factrecall_zh_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n现在请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=16),
+)
+
+LVEval_factrecall_zh_eval_cfg = dict(
+    evaluator=dict(type=LVEvalF1Evaluator, language='zh'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_factrecall_zh_datasets = [
+    dict(
+        type=LVEvalfactrecallzhDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_factrecall_zh_reader_cfg,
+        infer_cfg=LVEval_factrecall_zh_infer_cfg,
+        eval_cfg=LVEval_factrecall_zh_eval_cfg,
+    )
+    for name_len in get_dataset_names('factrecall_zh', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalhotpotwikiqa_mixup/lveval_hotpotwikiqa_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_hotpotwikiqa_mixup_gen_77ce82 import (
+        LVEval_hotpotwikiqa_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalhotpotwikiqa_mixup/lveval_hotpotwikiqa_mixup_gen_77ce82.py

@@ -0,0 +1,59 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LVEvalOPTF1Evaluator,
+    LVEvalhotpotwikiqaDataset,
+)
+
+LVEval_hotpotwikiqa_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_hotpotwikiqa_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Questions and answers are only relevant to some passages. Only give me the answer and do not output any other explanation and evidence.\n\nArticle: {context}\n\nPlease answer the following question based on the above passages. Questions and answers are only relevant to some passages. Only give me the answer and do not output any other explanation and evidence.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_hotpotwikiqa_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_hotpotwikiqa_mixup_datasets = [
+    dict(
+        type=LVEvalhotpotwikiqaDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_hotpotwikiqa_mixup_reader_cfg,
+        infer_cfg=LVEval_hotpotwikiqa_mixup_infer_cfg,
+        eval_cfg=LVEval_hotpotwikiqa_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names(
+        'hotpotwikiqa_mixup', DATASET_LENGTH_LEVEL
+    )
+]

build/lib/opencompass/configs/datasets/lveval/lvevallic_mixup/lveval_lic_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_lic_mixup_gen_01eb0c import (
+        LVEval_lic_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevallic_mixup/lveval_lic_mixup_gen_01eb0c.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTF1Evaluator, LVEvallicDataset
+
+LVEval_lic_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_lic_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='请根据下面给定的文章回答问题，问题和答案只与其中一篇文章有关。\n\n文章：{context}\n\n请现在基于上述文章回答下面的问题，问题和答案只与其中一篇文章有关。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_lic_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='zh'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_lic_mixup_datasets = [
+    dict(
+        type=LVEvallicDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_lic_mixup_reader_cfg,
+        infer_cfg=LVEval_lic_mixup_infer_cfg,
+        eval_cfg=LVEval_lic_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('lic_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_CR_mixup/lveval_loogle_CR_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_loogle_CR_mixup_gen_d7ea36 import (
+        LVEval_loogle_CR_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_CR_mixup/lveval_loogle_CR_mixup_gen_d7ea36.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTF1Evaluator, LVEvallooglecrDataset
+
+LVEval_loogle_CR_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_loogle_CR_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please answer the following question based on the given passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nArticle: {context}\n\nPlease answer the following question based on the above passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_loogle_CR_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_loogle_CR_mixup_datasets = [
+    dict(
+        type=LVEvallooglecrDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_loogle_CR_mixup_reader_cfg,
+        infer_cfg=LVEval_loogle_CR_mixup_infer_cfg,
+        eval_cfg=LVEval_loogle_CR_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('loogle_CR_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_MIR_mixup/lveval_loogle_MIR_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_loogle_MIR_mixup_gen_d7ea36 import (
+        LVEval_loogle_MIR_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_MIR_mixup/lveval_loogle_MIR_mixup_gen_d7ea36.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTF1Evaluator, LVEvallooglemirDataset
+
+LVEval_loogle_MIR_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_loogle_MIR_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please answer the following question based on the given passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nArticle: {context}\n\nPlease answer the following question based on the above passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_loogle_MIR_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_loogle_MIR_mixup_datasets = [
+    dict(
+        type=LVEvallooglemirDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_loogle_MIR_mixup_reader_cfg,
+        infer_cfg=LVEval_loogle_MIR_mixup_infer_cfg,
+        eval_cfg=LVEval_loogle_MIR_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('loogle_MIR_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_SD_mixup/lveval_loogle_SD_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_loogle_SD_mixup_gen_d7ea36 import (
+        LVEval_loogle_SD_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalloogle_SD_mixup/lveval_loogle_SD_mixup_gen_d7ea36.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LVEvalOPTF1Evaluator, LVEvallooglesdDataset
+
+LVEval_loogle_SD_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_loogle_SD_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please answer the following question based on the given passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nArticle: {context}\n\nPlease answer the following question based on the above passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_loogle_SD_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_loogle_SD_mixup_datasets = [
+    dict(
+        type=LVEvallooglesdDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_loogle_SD_mixup_reader_cfg,
+        infer_cfg=LVEval_loogle_SD_mixup_infer_cfg,
+        eval_cfg=LVEval_loogle_SD_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names('loogle_SD_mixup', DATASET_LENGTH_LEVEL)
+]

build/lib/opencompass/configs/datasets/lveval/lvevalmultifieldqa_en_mixup/lveval_multifieldqa_en_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_multifieldqa_en_mixup_gen_d7ea36 import (
+        LVEval_multifieldqa_en_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalmultifieldqa_en_mixup/lveval_multifieldqa_en_mixup_gen_d7ea36.py

@@ -0,0 +1,59 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LVEvalOPTF1Evaluator,
+    LVEvalmultifieldqaenDataset,
+)
+
+LVEval_multifieldqa_en_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_multifieldqa_en_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Please answer the following question based on the given passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nArticle: {context}\n\nPlease answer the following question based on the above passages. Questions and answers are only relevant to one passage. Only give me the answer and do not output any other explanation and evidence.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_multifieldqa_en_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='en'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_multifieldqa_en_mixup_datasets = [
+    dict(
+        type=LVEvalmultifieldqaenDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_multifieldqa_en_mixup_reader_cfg,
+        infer_cfg=LVEval_multifieldqa_en_mixup_infer_cfg,
+        eval_cfg=LVEval_multifieldqa_en_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names(
+        'multifieldqa_en_mixup', DATASET_LENGTH_LEVEL
+    )
+]

build/lib/opencompass/configs/datasets/lveval/lvevalmultifieldqa_zh_mixup/lveval_multifieldqa_zh_mixup_gen.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lveval_multifieldqa_zh_mixup_gen_0fbdad import (
+        LVEval_multifieldqa_zh_mixup_datasets,
+    )  # noqa: F401, F403

build/lib/opencompass/configs/datasets/lveval/lvevalmultifieldqa_zh_mixup/lveval_multifieldqa_zh_mixup_gen_0fbdad.py

@@ -0,0 +1,59 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LVEvalOPTF1Evaluator,
+    LVEvalmultifieldqazhDataset,
+)
+
+LVEval_multifieldqa_zh_mixup_reader_cfg = dict(
+    input_columns=['context', 'input'],
+    output_column='answers',
+    train_split='test',
+    test_split='test',
+)
+
+LVEval_multifieldqa_zh_mixup_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='请阅读以下文章并用中文回答问题，问题和答案只与其中一篇文章有关。只需要直接给出问题的答案，不要输出其他任何解释和证据。\n\n文章：{context}\n\n请基于上面的文章回答下面的问题，问题和答案只与其中一篇文章有关。只需要直接给出问题的答案，不要输出其他任何解释和证据。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=64),
+)
+
+LVEval_multifieldqa_zh_mixup_eval_cfg = dict(
+    evaluator=dict(type=LVEvalOPTF1Evaluator, language='zh'), pred_role='BOT'
+)
+
+DATASET_LENGTH_LEVEL = ['16k', '32k', '64k', '128k', '256k']
+
+
+def get_dataset_names(dataset_name, length_levels):
+    datasets = []
+    for length in length_levels:
+        datasets.append(f'{dataset_name}_{length}')
+    return datasets
+
+
+LVEval_multifieldqa_zh_mixup_datasets = [
+    dict(
+        type=LVEvalmultifieldqazhDataset,
+        abbr='LVEval_' + name_len,
+        path='Infinigence/LVEval',
+        name=name_len,
+        reader_cfg=LVEval_multifieldqa_zh_mixup_reader_cfg,
+        infer_cfg=LVEval_multifieldqa_zh_mixup_infer_cfg,
+        eval_cfg=LVEval_multifieldqa_zh_mixup_eval_cfg,
+    )
+    for name_len in get_dataset_names(
+        'multifieldqa_zh_mixup', DATASET_LENGTH_LEVEL
+    )
+]

build/lib/opencompass/configs/datasets/needlebench/atc/atc.py

@@ -0,0 +1,110 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.needlebench.atc import NeedleBenchATCOrderedDataset
+from opencompass.datasets.needlebench.atc import NeedleBenchATCDataset
+from opencompass.datasets.needlebench.origin import NeedleBenchOriginEvaluator
+from opencompass.datasets.needlebench.origin import needlebench_postprocess
+from opencompass.datasets.needlebench.origin import needlebench_dataset_postprocess
+
+needlebench_reader_cfg = dict(input_columns=['prompt'], output_column='answer')
+
+needlebench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+                dict(role='BOT', prompt='{answer}\n'),
+            ]
+        )
+        ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+needlebench_eval_cfg = dict(
+    evaluator=dict(type=NeedleBenchOriginEvaluator),
+    pred_postprocessor=dict(type=needlebench_postprocess),
+    dataset_postprocessor=dict(type=needlebench_dataset_postprocess),
+    pred_role='BOT')
+
+needle_num_list = list(range(2, 100, 3))
+document_depth_percent_intervals = 20
+repeats = 30
+
+path = 'opencompass/needlebench'
+file_name = 'names.json'
+
+needlebench_atc_datasets_zh = []
+needlebench_atc_datasets_en = []
+needlebench_atc_datasets_zh_ordered = []
+needlebench_atc_datasets_en_ordered = []
+
+for num_needles in needle_num_list:
+    # ordered English version
+    dataset_dict = {
+        'abbr': f'needlebench_atc_challenge'
+                f'needle_{num_needles}_en_ordered',
+        'type': NeedleBenchATCOrderedDataset,
+        'path': path,
+        'file_name': file_name,
+        'num_needles': num_needles,
+        'language': 'English',
+        'repeats': repeats,
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg
+    }
+    needlebench_atc_datasets_en_ordered.append(dataset_dict)
+
+
+for num_needles in needle_num_list:
+    # ordered Chinese version
+    dataset_dict = {
+        'abbr': f'needlebench_atc_challenge'
+                f'needle_{num_needles}_zh_ordered',
+        'type': NeedleBenchATCOrderedDataset,
+        'path': path,
+        'file_name': file_name,
+        'num_needles': num_needles,
+        'language': 'Chinese',
+        'repeats': repeats,
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg
+    }
+    needlebench_atc_datasets_zh_ordered.append(dataset_dict)
+
+for num_needles in needle_num_list:
+    # standard English version
+    dataset_dict = {
+        'abbr': f'needlebench_atc_challenge'
+                f'needle_{num_needles}_en',
+        'type': NeedleBenchATCDataset,
+        'path': path,
+        'file_name': file_name,
+        'num_needles': num_needles,
+        'language': 'English',
+        'repeats': repeats,
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg
+    }
+    needlebench_atc_datasets_en.append(dataset_dict)
+
+for num_needles in needle_num_list:
+    # standard Chinese version
+    dataset_dict = {
+        'abbr': f'needlebench_atc_challenge'
+                f'needle_{num_needles}_zh',
+        'type': NeedleBenchATCDataset,
+        'path': path,
+        'file_name': file_name,
+        'num_needles': num_needles,
+        'language': 'Chinese',
+        'repeats': repeats,
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg
+    }
+    needlebench_atc_datasets_zh.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice.py

@@ -0,0 +1,136 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator
+from opencompass.datasets.needlebench.atc_choice import NeedleBenchATCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+# ----------------------- Prompt Content----------------------- #
+
+few_shot_prompts = {
+    'single_choice_prompts': {
+        'single_choice_cn': [
+            dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王展\n'),
+            dict(role='BOT', prompt='回答: A'),
+            dict(role='HUMAN', prompt='问题: 梅建平作为黄玉兰的姥姥，对黄玉兰的成长有重要影响。梅建平的妈妈是朱丽丽。蓝鑫把黄玉兰视为其母亲。焦慧不仅是朱丽丽的妈妈，还是朱丽丽的榜样。 \n在上面提供的打乱的家族关系文本中，"蓝鑫"的能够向上追溯到的最年长的亲人是谁？\nA. 梅建平\nB. 朱丽丽\nC. 蓝鑫\nD. 焦慧\n'),
+            dict(role='BOT', prompt='回答: D'),
+            dict(role='HUMAN', prompt='问题: 毕东把柳金凤视为其姥姥。奉兵作为柳金凤的妈妈，对柳金凤的成长有重要影响。余萍把杨颖视为其爸爸。毕东在郭建华的生命中扮演着父亲的角色。常宁的外公是余萍。刘慧是郭建华所生的孩子。刘慧在杨颖的生命中扮演着外公的角色。 \n在上面提供的打乱的家族关系文本中，"常宁"的能够向上追溯到的最年长的亲人是谁？\nA. 柳金凤\nB. 毕东\nC. 奉兵\nD. 余萍\n'),
+            dict(role='BOT', prompt='回答: C'),
+            dict(role='HUMAN', prompt='问题: 魏丽丽在谢平的生命中扮演着奶奶的角色。郭兵是魏阳的姥姥。谢平是郑玉珍的外婆。丁颖把武波视为其外公。丁颖在李建国的生命中扮演着外婆的角色。武波的父亲是刘瑜。许玲把余桂芳视为其父亲。刘瑜把许玲视为其爷爷。李建国对郭兵来说，不只是一个爷爷，还是一个朋友。魏丽丽的外公是魏阳。 \n在上面提供的打乱的家族关系文本中，"郑玉珍"的能够向上追溯到的最年长的亲人是谁？\nA. 魏丽丽\nB. 刘瑜\nC. 李建国\nD. 余桂芳\n'),
+            dict(role='BOT', prompt='回答: D'),
+            dict(role='HUMAN', prompt='问题: {question}'),
+        ],
+        'single_choice_en': [
+            dict(role='HUMAN', prompt="Question: Jasmine Lane plays the role of James Hill's father in James Hill's life. \nGiven the scrambled family relationships described above, who is the eldest relative that 'James Hill' can trace back to in the context?\nA. Jasmine Lane\nB. James Hill\nC. Christopher Flores\nD. Paula Reeves\n"),
+            dict(role='BOT', prompt='Answer: A'),
+            dict(role='HUMAN', prompt="Question: For Michael Goodwin, Annette Valdez is not just a dad, but also a friend.For Annette Valdez, Mary Sanders is not just a father, but also a friend. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Michael Goodwin' can trace back to in the context?\nA. Michael Goodwin\nB. Annette Valdez\nC. Mary Sanders\nD. James Mitchell\n"),
+            dict(role='BOT', prompt='Answer: C'),
+            dict(role='HUMAN', prompt="Question: Laura Garcia considers Mr. Stephen Valdez DDS as their mom.Mr. Stephen Valdez DDS's maternal grandfather is Elizabeth Smith.Laura Garcia is Ashley Johnson's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Ashley Johnson' can trace back to in the context?\nA. Ashley Johnson\nB. Mr. Stephen Valdez DDS\nC. Laura Garcia\nD. Elizabeth Smith\n"),
+            dict(role='BOT', prompt='Answer: D'),
+            dict(role='HUMAN', prompt="Question: Justin Sanchez's paternal grandfather is Laurie Bean.For Laurie Bean, Christina Kim is not just a mother, but also a friend.Jessica Bass, as Christina Kim's maternal grandfather, has a significant impact on Christina Kim's upbringing.Sarah Hampton considers Justin Sanchez as their father. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Sarah Hampton' can trace back to in the context?\nA. Christina Kim\nB. Jessica Bass\nC. Laurie Bean\nD. Justin Sanchez\n"),
+            dict(role='BOT', prompt='Answer: B'),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+        'single_choice_cn_reasoning': [
+            dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王���\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 刘红的爸爸是毕东。\n\n因此，"刘红"能够向上追溯到的最年长的亲人是毕东。所以答案是：A'),
+
+            dict(role='HUMAN', prompt='问题: 梅建平作为黄玉兰的姥姥，对黄玉兰的成长有重要影响。梅建平的妈妈是朱丽丽。蓝鑫把黄玉兰视为其母亲。焦慧不仅是朱丽丽的妈妈，还是朱丽丽的榜样。 \n在上面提供的打乱的家族关系文本中，"蓝鑫"的能够向上追溯到的最年长的亲人是谁？\nA. 梅建平\nB. 朱丽丽\nC. 蓝鑫\nD. 焦慧\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 蓝鑫把黄玉兰视为其母亲。\n- 黄玉兰的姥姥是梅建平。\n- 梅建平的妈妈是朱丽丽。\n- 朱丽丽的妈妈是焦慧。\n\n因此，"蓝鑫"能够向上追溯到的最年长的亲人是焦慧。所以答案是：D'),
+
+            dict(role='HUMAN', prompt='问题: 毕东把柳金凤视为其姥姥。奉兵作为柳金凤的妈妈，对柳金凤的成长有重要影响。余萍把杨颖视为其爸爸。毕东在郭建华的生命中扮演着父亲的角色。常宁的外公是余萍。刘慧是郭建华所生的孩子。刘慧在杨颖的生命中扮演着外公的角色。 \n在上面提供的打乱的家族关系文本中，"常宁"的能够向上追溯到的最年长的亲人是谁？\nA. 柳金凤\nB. 毕东\nC. 奉兵\nD. 余萍\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 常宁的外公是余萍。\n- 余萍把杨颖视为其爸爸。\n- 杨颖的外公是刘慧。\n- 刘慧是郭建华所生的孩子。\n- 郭建华的父亲是毕东。\n- 毕东的姥姥是柳金凤。\n- 柳金凤的妈妈是奉兵。\n\n因此，"常宁"能够向上追溯到的最年长的亲人是奉兵。所以答案是：C'),
+
+            dict(role='HUMAN', prompt='问题: 魏丽丽在谢平的生命中扮演着奶奶的角色。郭兵是魏阳的姥姥。谢平是郑玉珍的外婆。丁颖把武波视为其外公。丁颖在李建国的生命中扮演着外婆的角色。武波的父亲是刘瑜。许玲把余桂芳视为其父亲。刘瑜把许玲视为其爷爷。李建国对郭兵来说，不只是一个爷爷，还是一个朋友。魏丽丽的外公是魏阳。 \n在上面提供的打乱的家族关系文本中，"郑玉珍"的能够向上追溯到的最年长的亲人是谁？\nA. 魏丽丽\nB. 刘瑜\nC. 李建国\nD. 余桂芳\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 郑玉珍的外婆是谢平。\n- 谢平的奶奶是魏丽丽。\n- 魏丽丽的外公是魏阳。 \n- 魏阳的姥姥是郭兵。\n- 郭兵的爷爷是李建国。 \n- 李建国的外婆是丁颖。 \n- 丁颖的外公是武波。 \n- 武波的父亲是刘瑜。\n- 刘瑜的爷爷是许玲。\n- 许玲的父亲是余桂芳。 \n\n因此，"郑玉珍"能够向上追溯到的最年长的亲人是余桂芳。所以答案是：D'),
+            dict(role='HUMAN', prompt='问题: {question}'),
+        ],
+        'single_choice_en_reasoning': [
+            dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
+            dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D"),
+            dict(role='HUMAN', prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B"),
+            dict(role='HUMAN', prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A"),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+    },
+}
+
+# ----------------------- Prompt Settings ----------------------- #
+needle_num_list = list(range(2, 20, 1))
+path = 'opencompass/needlebench'
+file_name = 'names.json'
+
+repeats = 10
+
+# Use Zero-Shot or not
+with_few_shot = True
+
+# Max for this dataset is 4, should be set with `with_few_shot`
+few_shot_samples = 4
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+needlebench_prompts = few_shot_prompts
+single_choice_prompts = needlebench_prompts['single_choice_prompts']
+
+# Set few shot prompt number
+for _name in list(single_choice_prompts.keys()):
+    if with_few_shot:
+        assert few_shot_samples > 0 and few_shot_samples <= 4
+        single_choice_prompts[_name] = \
+            single_choice_prompts[_name][- few_shot_samples * 2 - 1:]
+
+# ----------------------- Dataset Settings ----------------------- #
+
+needlebench_datasets = []
+
+
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer,),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+    current_needle_num_list = needle_num_list
+
+    for num_needles in current_needle_num_list:
+        abbr = (f'NeedleBenchATCDataset-'
+                f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name': file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice_20.py

@@ -0,0 +1,133 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator
+from opencompass.datasets.needlebench.atc_choice import NeedleBenchATCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+# ----------------------- Prompt Content----------------------- #
+
+few_shot_prompts = {
+    'single_choice_prompts': {
+        'single_choice_cn': [
+            dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王展\n'),
+            dict(role='BOT', prompt='回答: A'),
+            dict(role='HUMAN', prompt='问题: 梅建平作为黄玉兰的姥姥，对黄玉兰的成长有重要影响。梅建平的妈妈是朱丽丽。蓝鑫把黄玉兰视为其母亲。焦慧不仅是朱丽丽的妈妈，还是朱丽丽的榜样。 \n在上面提供的打乱的家族关系文本中，"蓝鑫"的能够向上追溯到的最年长的亲人是谁？\nA. 梅建平\nB. 朱丽丽\nC. 蓝鑫\nD. 焦慧\n'),
+            dict(role='BOT', prompt='回答: D'),
+            dict(role='HUMAN', prompt='问题: 毕东把柳金凤视为其姥姥。奉兵作为柳金凤的妈妈，对柳金凤的成长有重要影响。余萍把杨颖视为其爸爸。毕东在郭建华的生命中扮演着父亲的角色。常宁的外公是余萍。刘慧是郭建华所生的孩子。刘慧在杨颖的生命中扮演着外公的角色。 \n在上面提供的打乱的家族关系文本中，"常宁"的能够向上追溯到的最年长的亲人是谁？\nA. 柳金凤\nB. 毕东\nC. 奉兵\nD. 余萍\n'),
+            dict(role='BOT', prompt='回答: C'),
+            dict(role='HUMAN', prompt='问题: 魏丽丽在谢平的生命中扮演着奶奶的角色。郭兵是魏阳的姥姥。谢平是郑玉珍的外婆。丁颖把武波视为其外公。丁颖在李建国的生命中扮演着外婆的角色。武波的父亲是刘瑜。许玲把余桂芳视为其父亲。刘瑜把许玲视为其爷爷。李建国对郭兵来说，不只是一个爷爷，还是一个朋友。魏丽丽的外公是魏阳。 \n在上面提供的打乱的家族关系文本中，"郑玉珍"的能够向上追溯到的最年长的亲人是谁？\nA. 魏丽丽\nB. 刘瑜\nC. 李建国\nD. 余桂芳\n'),
+            dict(role='BOT', prompt='回答: D'),
+            dict(role='HUMAN', prompt='问题: {question}'),
+        ],
+        'single_choice_en': [
+            dict(role='HUMAN', prompt="Question: Jasmine Lane plays the role of James Hill's father in James Hill's life. \nGiven the scrambled family relationships described above, who is the eldest relative that 'James Hill' can trace back to in the context?\nA. Jasmine Lane\nB. James Hill\nC. Christopher Flores\nD. Paula Reeves\n"),
+            dict(role='BOT', prompt='Answer: A'),
+            dict(role='HUMAN', prompt="Question: For Michael Goodwin, Annette Valdez is not just a dad, but also a friend.For Annette Valdez, Mary Sanders is not just a father, but also a friend. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Michael Goodwin' can trace back to in the context?\nA. Michael Goodwin\nB. Annette Valdez\nC. Mary Sanders\nD. James Mitchell\n"),
+            dict(role='BOT', prompt='Answer: C'),
+            dict(role='HUMAN', prompt="Question: Laura Garcia considers Mr. Stephen Valdez DDS as their mom.Mr. Stephen Valdez DDS's maternal grandfather is Elizabeth Smith.Laura Garcia is Ashley Johnson's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Ashley Johnson' can trace back to in the context?\nA. Ashley Johnson\nB. Mr. Stephen Valdez DDS\nC. Laura Garcia\nD. Elizabeth Smith\n"),
+            dict(role='BOT', prompt='Answer: D'),
+            dict(role='HUMAN', prompt="Question: Justin Sanchez's paternal grandfather is Laurie Bean.For Laurie Bean, Christina Kim is not just a mother, but also a friend.Jessica Bass, as Christina Kim's maternal grandfather, has a significant impact on Christina Kim's upbringing.Sarah Hampton considers Justin Sanchez as their father. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Sarah Hampton' can trace back to in the context?\nA. Christina Kim\nB. Jessica Bass\nC. Laurie Bean\nD. Justin Sanchez\n"),
+            dict(role='BOT', prompt='Answer: B'),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+        'single_choice_cn_reasoning': [
+            dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王���\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 刘红的爸爸是毕东。\n\n因此，"刘红"能够向上追溯到的最年长的亲人是毕东。所以答案是：A'),
+
+            dict(role='HUMAN', prompt='问题: 梅建平作为黄玉兰的姥姥，对黄玉兰的成长有重要影响。梅建平的妈妈是朱丽丽。蓝鑫把黄玉兰视为其母亲。焦慧不仅是朱丽丽的妈妈，还是朱丽丽的榜样。 \n在上面提供的打乱的家族关系文本中，"蓝鑫"的能够向上追溯到的最年长的亲人是谁？\nA. 梅建平\nB. 朱丽丽\nC. 蓝鑫\nD. 焦慧\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 蓝鑫把黄玉兰视为其母亲。\n- 黄玉兰的姥姥是梅建平。\n- 梅建平的妈妈是朱丽丽。\n- 朱丽丽的妈妈是焦慧。\n\n因此，"蓝鑫"能够向上追溯到的最年长的亲人是焦慧。所以答案是：D'),
+
+            dict(role='HUMAN', prompt='问题: 毕东把柳金凤视为其姥姥。奉兵作为柳金凤的妈妈，对柳金凤的成长有重要影响。余萍把杨颖视为其爸爸。毕东在郭建华的生命中扮演着父亲的角色。常宁的外公是余萍。刘慧是郭建华所生的孩子。刘慧在杨颖的生命中扮演着外公的角色。 \n在上面提供的打乱的家族关系文本中，"常宁"的能够向上追溯到的最年长的亲人是谁？\nA. 柳金凤\nB. 毕东\nC. 奉兵\nD. 余萍\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 常宁的外公是余萍。\n- 余萍把杨颖视为其爸爸。\n- 杨颖的外公是刘慧。\n- 刘慧是郭建华所生的孩子。\n- 郭建华的父亲是毕东。\n- 毕东的姥姥是柳金凤。\n- 柳金凤的妈妈是奉兵。\n\n因此，"常宁"能够向上追溯到的最年长的亲人是奉兵。所以答案是：C'),
+
+            dict(role='HUMAN', prompt='问题: 魏丽丽在谢平的生命中扮演着w奶奶的角色。郭兵是魏阳的姥姥。谢平是郑玉珍的外婆。丁颖把武波视为其外公。丁颖在李建国的生命中扮演着外婆的角色。武波的父亲是刘瑜。许玲把余桂芳视为其父亲。刘瑜把许玲视为其爷爷。李建国对郭兵来说，不只是一个爷爷，还是一个朋友。魏丽丽的外公是魏阳。 \n在上面提供的打乱的家族关系文本中，"郑玉珍"的能够向上追溯到的最年长的亲人是谁？\nA. 魏丽丽\nB. 刘瑜\nC. 李建国\nD. 余桂芳\n'),
+            dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 郑玉珍的外婆是谢平。\n- 谢平的奶奶是魏丽丽。\n- 魏丽丽的外公是魏阳。 \n- 魏阳的姥姥是郭兵。\n- 郭兵的爷爷是李建国。 \n- 李建国的外婆是丁颖。 \n- 丁颖的外公是武波。 \n- 武波的父亲是刘瑜。\n- 刘瑜的爷爷是许玲。\n- 许玲的父亲是余桂芳。 \n\n因此，"郑玉珍"能够向上追溯到的最年长的亲人是余桂芳。所以答案是：D'),
+            dict(role='HUMAN', prompt='问题: {question}'),
+        ],
+        'single_choice_en_reasoning': [
+            dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
+            dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D"),
+            dict(role='HUMAN', prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B"),
+            dict(role='HUMAN', prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A"),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+    },
+}
+
+# ----------------------- Prompt Settings ----------------------- #
+needle_num_list = list(range(2, 20, 1))
+path = 'opencompass/needlebench'
+file_name = 'names.json'
+repeats = 10
+
+# Use Zero-Shot or not
+with_few_shot = True
+
+# Max for this dataset is 4, should be set with `with_few_shot`
+few_shot_samples = 4
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+needlebench_prompts = few_shot_prompts
+single_choice_prompts = needlebench_prompts['single_choice_prompts']
+
+# Set few shot prompt number
+for _name in list(single_choice_prompts.keys()):
+    if with_few_shot:
+        assert few_shot_samples > 0 and few_shot_samples <= 4
+        single_choice_prompts[_name] = \
+            single_choice_prompts[_name][- few_shot_samples * 2 - 1:]
+
+# ----------------------- Dataset Settings ----------------------- #
+
+needlebench_datasets = []
+
+
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer,),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+    for num_needles in needle_num_list:
+        abbr = (f'NeedleBenchATCDataset-'
+                f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name': file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice_50.py

@@ -0,0 +1,43 @@
+from mmengine.config import read_base
+with read_base():
+    from .atc_choice_20 import *
+
+needle_num_list = list(range(2, 50, 1))
+needlebench_datasets = []
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer,),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+    for num_needles in needle_num_list:
+        abbr = (f'NeedleBenchATCDataset-'
+                f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name': file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py

@@ -0,0 +1,125 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator
+from opencompass.datasets.needlebench.atc_choice import NeedleBenchATCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+# ----------------------- Prompt Content----------------------- #
+
+few_shot_prompts = {
+    'single_choice_prompts': {
+        'single_choice_en_reasoning': [
+            dict(
+                role='HUMAN',
+                prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n",
+            ),
+            dict(
+                role='BOT',
+                prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C",
+            ),
+            dict(
+                role='HUMAN',
+                prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n",
+            ),
+            dict(
+                role='BOT',
+                prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D",
+            ),
+            dict(
+                role='HUMAN',
+                prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n",
+            ),
+            dict(
+                role='BOT',
+                prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B",
+            ),
+            dict(
+                role='HUMAN',
+                prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n",
+            ),
+            dict(
+                role='BOT',
+                prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A",
+            ),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+    },
+}
+
+# ----------------------- Prompt Settings ----------------------- #
+needle_num_list = list(range(2, 50, 1))
+path = 'opencompass/needlebench'
+file_name = 'names.json'
+repeats = 10
+
+# Use Zero-Shot or not
+with_few_shot = True
+
+# Max for this dataset is 4, should be set with `with_few_shot`
+few_shot_samples = 4
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+needlebench_prompts = few_shot_prompts
+single_choice_prompts = needlebench_prompts['single_choice_prompts']
+
+# Set few shot prompt number
+for _name in list(single_choice_prompts.keys()):
+    if with_few_shot:
+        assert few_shot_samples > 0 and few_shot_samples <= 4
+        single_choice_prompts[_name] = single_choice_prompts[_name][
+            -few_shot_samples * 2 - 1 :
+        ]
+
+# ----------------------- Dataset Settings ----------------------- #
+
+needlebench_datasets = []
+
+
+needlebench_atc_reader_cfg = dict(input_columns=['question'], output_column='answer')
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(
+            type=GenInferencer,
+        ),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+    )
+
+    for num_needles in needle_num_list:
+        abbr = (
+            f'NeedleBenchATCDataset-'
+            f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}'
+        )
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name':file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg,
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice_80.py

@@ -0,0 +1,43 @@
+from mmengine.config import read_base
+with read_base():
+    from .atc_choice_20 import *
+
+needle_num_list = list(range(2, 80, 1))
+needlebench_datasets = []
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer,),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+    for num_needles in needle_num_list:
+        abbr = (f'NeedleBenchATCDataset-'
+                f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name': file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py

@@ -0,0 +1,97 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator
+from opencompass.datasets.needlebench.atc_choice import NeedleBenchATCDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+# ----------------------- Prompt Content----------------------- #
+
+few_shot_prompts = {
+    'single_choice_prompts': {
+        'single_choice_en_reasoning': [
+            dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
+            dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D"),
+            dict(role='HUMAN', prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B"),
+            dict(role='HUMAN', prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n"),
+            dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A"),
+            dict(role='HUMAN', prompt='Question: {question}'),
+        ],
+    },
+}
+
+# ----------------------- Prompt Settings ----------------------- #
+needle_num_list = list(range(2, 80, 1))
+path = 'opencompass/needlebench'
+file_name = 'names.json'
+repeats = 10
+
+# Use Zero-Shot or not
+with_few_shot = True
+
+# Max for this dataset is 4, should be set with `with_few_shot`
+few_shot_samples = 4
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+needlebench_prompts = few_shot_prompts
+single_choice_prompts = needlebench_prompts['single_choice_prompts']
+
+# Set few shot prompt number
+for _name in list(single_choice_prompts.keys()):
+    if with_few_shot:
+        assert few_shot_samples > 0 and few_shot_samples <= 4
+        single_choice_prompts[_name] = \
+            single_choice_prompts[_name][- few_shot_samples * 2 - 1:]
+
+# ----------------------- Dataset Settings ----------------------- #
+
+needlebench_datasets = []
+
+
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
+
+for _name in list(single_choice_prompts.keys()):
+
+    needlebench_atc_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=(single_choice_prompts[_name])),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer,),
+    )
+
+    needlebench_atc_eval_cfg = dict(
+        evaluator=dict(type=CircularEvaluator),
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+    for num_needles in needle_num_list:
+        abbr = (f'NeedleBenchATCDataset-'
+                f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
+        language = 'English' if 'en' in _name else 'Chinese'
+        if 'reasoning' in _name:
+            abbr += '-Reasoning'
+        dataset_dict = {
+            'abbr': abbr,
+            'type': NeedleBenchATCDataset,
+            'path': path,
+            'file_name': file_name,
+            'num_needles': num_needles,
+            'language': language,
+            'repeats': repeats,
+            'with_circular': with_circular_eval,
+            'reader_cfg': needlebench_atc_reader_cfg,
+            'infer_cfg': needlebench_atc_infer_cfg,
+            'eval_cfg': needlebench_atc_eval_cfg
+        }
+        needlebench_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_1000k.py

@@ -0,0 +1,18 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .needlebench_multi_reasoning_1000k import needlebench_2needle_en_datasets as needlebench_multi_2needle_en_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_3needle_en_datasets as needlebench_multi_3needle_en_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_4needle_en_datasets as needlebench_multi_4needle_en_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_5needle_en_datasets as needlebench_multi_5needle_en_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_2needle_zh_datasets as needlebench_multi_2needle_zh_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_3needle_zh_datasets as needlebench_multi_3needle_zh_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_4needle_zh_datasets as needlebench_multi_4needle_zh_datasets
+    from .needlebench_multi_reasoning_1000k import needlebench_5needle_zh_datasets as needlebench_multi_5needle_zh_datasets
+
+    from .needlebench_single_1000k import needlebench_en_datasets as needlebench_origin_en_datasets
+    from .needlebench_single_1000k import needlebench_zh_datasets as needlebench_origin_zh_datasets
+    from .needlebench_multi_retrieval_1000k import needlebench_en_datasets as needlebench_parallel_en_datasets
+    from .needlebench_multi_retrieval_1000k import needlebench_zh_datasets as needlebench_parallel_zh_datasets
+
+needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

build/lib/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py

@@ -0,0 +1,288 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.needlebench.multi import NeedleBenchMultiDataset
+from opencompass.datasets.needlebench.multi import NeedleBenchMultiEvaluator
+from opencompass.datasets.needlebench.origin import needlebench_postprocess
+from opencompass.datasets.needlebench.origin import needlebench_dataset_postprocess
+import math
+
+
+def logistic(x, L=100, x0=50, k=0.1):
+    return round(L / (1 + math.exp(-k * (x - x0))), 3)
+
+
+def generate_linear_space(start, end, num):
+    if num == 1:
+        return [start]
+    elif num < 1:
+        raise ValueError('num must be at least 1.')
+    step = (end - start) / (num - 1)
+    return [start + step * i for i in range(num)]
+
+
+def generate_depth_percents(intervals, interval_type):
+    if interval_type == 'linear':
+        return generate_linear_space(0, 100, intervals)
+    elif interval_type == 'sigmoid':
+        linear_space = generate_linear_space(0, 100, intervals)
+        return [logistic(x) for x in linear_space]
+    else:
+        raise ValueError('Unsupported interval type')
+
+
+needlebench_reader_cfg = dict(input_columns=['prompt'], output_column='answer')
+
+needlebench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+                dict(role='BOT', prompt='{answer}\n'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+needlebench_eval_cfg = dict(
+    evaluator=dict(type=NeedleBenchMultiEvaluator),
+    pred_postprocessor=dict(type=needlebench_postprocess),
+    dataset_postprocessor=dict(type=needlebench_dataset_postprocess),
+    pred_role='BOT',
+)
+
+context_lengths = [20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000]
+depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
+
+# ----------English Version----------
+base_path = 'opencompass/needlebench'
+file_list = ['PaulGrahamEssays.jsonl']
+
+needle_file_name = 'multi_needle_reasoning_en.json'
+diff = 10
+num_needles = 2
+needlebench_2needle_en_datasets = []
+language = 'English'
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 600,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_2needle_en_datasets.append(dataset_dict)
+
+num_needles = 3
+needlebench_3needle_en_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 600,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_3needle_en_datasets.append(dataset_dict)
+
+num_needles = 4
+needlebench_4needle_en_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 600,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_4needle_en_datasets.append(dataset_dict)
+
+num_needles = 5
+needlebench_5needle_en_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 600,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_5needle_en_datasets.append(dataset_dict)
+
+# ----------Chinese Version----------
+base_path = 'opencompass/needlebench'
+file_list = ['zh_finance.jsonl']
+
+needle_file_name = 'multi_needle_reasoning_zh.json'
+diff = 10
+num_needles = 2
+needlebench_2needle_zh_datasets = []
+language = 'Chinese'
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 200,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_2needle_zh_datasets.append(dataset_dict)
+
+num_needles = 3
+needlebench_3needle_zh_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 200,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_3needle_zh_datasets.append(dataset_dict)
+
+num_needles = 4
+needlebench_4needle_zh_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 200,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_4needle_zh_datasets.append(dataset_dict)
+
+num_needles = 5
+needlebench_5needle_zh_datasets = []
+
+for original_context_length in context_lengths:
+    for depth_percent in depths_list:
+        dataset_dict = {
+            'abbr': f'Length{original_context_length}'
+            f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k',
+            'type': NeedleBenchMultiDataset,
+            'path': base_path,
+            'length': original_context_length,
+            'depth': int(depth_percent),
+            'tokenizer_model': 'gpt-4',
+            'file_list': file_list,
+            'num_repeats_per_file': 10,
+            'length_buffer': 200,
+            'guide': True,
+            'language': language,
+            'needle_file_name': needle_file_name,
+            'num_needles': num_needles,
+            'diff': diff,
+            'reader_cfg': needlebench_reader_cfg,
+            'infer_cfg': needlebench_infer_cfg,
+            'eval_cfg': needlebench_eval_cfg,
+        }
+        needlebench_5needle_zh_datasets.append(dataset_dict)

build/lib/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py

@@ -0,0 +1,108 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.needlebench.parallel import NeedleBenchParallelDataset
+from opencompass.datasets.needlebench.parallel import NeedleBenchParallelEvaluator
+from opencompass.datasets.needlebench.origin import needlebench_postprocess
+from opencompass.datasets.needlebench.origin import needlebench_dataset_postprocess
+import math
+
+
+def logistic(x, L=100, x0=50, k=0.1):
+    return round(L / (1 + math.exp(-k * (x - x0))), 3)
+
+
+def generate_linear_space(start, end, num):
+    if num == 1:
+        return [start]
+    elif num < 1:
+        raise ValueError('num must be at least 1.')
+    step = (end - start) / (num - 1)
+    return [start + step * i for i in range(num)]
+
+
+def generate_depth_percents(intervals, interval_type):
+    if interval_type == 'linear':
+        return generate_linear_space(0, 100, intervals)
+    elif interval_type == 'sigmoid':
+        linear_space = generate_linear_space(0, 100, intervals)
+        return [logistic(x) for x in linear_space]
+    else:
+        raise ValueError('Unsupported interval type')
+
+
+needlebench_reader_cfg = dict(input_columns=['prompt'], output_column='answer')
+
+needlebench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{prompt}'),
+                dict(role='BOT', prompt='{answer}\n'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+needlebench_eval_cfg = dict(
+    evaluator=dict(type=NeedleBenchParallelEvaluator),
+    pred_postprocessor=dict(type=needlebench_postprocess),
+    dataset_postprocessor=dict(type=needlebench_dataset_postprocess),
+    pred_role='BOT',
+)
+
+context_lengths = list([20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000])
+document_depth_percent_intervals = 20
+document_depth_percent_interval_type = 'linear'
+
+base_path = 'opencompass/needlebench'
+file_list = ['PaulGrahamEssays.jsonl']
+needlebench_en_datasets = []
+needle_file_name = 'needles.jsonl'
+depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
+
+for original_context_length in context_lengths:
+    dataset_dict = {
+        'abbr': f'Length{original_context_length}' f'_parallel_en_1000k',
+        'type': NeedleBenchParallelDataset,
+        'path': base_path,
+        'needle_file_name': needle_file_name,
+        'length': original_context_length,
+        'depths': depths,
+        'tokenizer_model': 'gpt-4',
+        'file_list': file_list,
+        'num_repeats_per_file': 25,
+        'length_buffer': 3000,
+        'guide': True,
+        'language': 'English',
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg,
+    }
+    needlebench_en_datasets.append(dataset_dict)
+
+file_list = ['zh_finance.jsonl']
+needlebench_zh_datasets = []
+
+for original_context_length in context_lengths:
+    dataset_dict = {
+        'abbr': f'Length{original_context_length}' f'_parallel_zh_1000k',
+        'type': NeedleBenchParallelDataset,
+        'path': base_path,
+        'needle_file_name': needle_file_name,
+        'length': original_context_length,
+        'depths': depths,
+        'tokenizer_model': 'gpt-4',
+        'file_list': file_list,
+        'num_repeats_per_file': 25,
+        'length_buffer': 200,
+        'guide': True,
+        'language': 'Chinese',
+        'reader_cfg': needlebench_reader_cfg,
+        'infer_cfg': needlebench_infer_cfg,
+        'eval_cfg': needlebench_eval_cfg,
+    }
+    needlebench_zh_datasets.append(dataset_dict)