Add files using upload-large-folder tool

tests/dataset/test_humaneval.py

@@ -0,0 +1,110 @@
+import unittest
+
+from opencompass.datasets.humaneval import humaneval_postprocess
+
+
+def run_humaneval_check(completion):
+    program = [
+        'def get_fraction(x: float) -> float:',
+        humaneval_postprocess(completion),
+        '',
+        'assert get_fraction(1.28) == 0.28',
+        'assert get_fraction(1.0) == 0.0',
+    ]
+    program = '\n'.join(program)
+    exec(program)
+
+
+class TestHumaneval(unittest.TestCase):
+
+    def test_vanilla(self):
+        raw = '    return x - int(x)'
+        run_humaneval_check(raw)
+
+    def test_python_quote(self):
+        lines = [
+            '```python',
+            '    return x - int(x)',
+            '```',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_bare_quote(self):
+        lines = [
+            '```',
+            '    return x - int(x)',
+            '```',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_error_space_quote(self):
+        lines = [
+            '```',
+            '  return x - int(x)',
+            '```',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_import_1(self):
+        lines = [
+            'import numpy as np',
+            'import math',
+            'from typing import List',
+            '',
+            'def func(x):',
+            '    return x - int(x)',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_import_2(self):
+        lines = [
+            'from typing import List',
+            'import numpy as np',
+            'import math',
+            'def func(x):',
+            '    return x - int(x)',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_import_3(self):
+        lines = [
+            'import math',
+            '',
+            '',
+            'def func(x):',
+            '    return x - int(x)',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_comment(self):
+        lines = [
+            'def func(x: float) -> float:',
+            "    '''",
+            '    blah blah blah',
+            '    blah blah blah',
+            "    '''",
+            '    return x - int(x)',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)
+
+    def test_additional(self):
+        lines = [
+            '    return x - int(x)',
+            '',
+            '',
+            'def func(x: float) -> float:',
+            "    '''",
+            '    blah blah blah',
+            '    blah blah blah',
+            "    '''",
+            '    return x - int(x)',
+        ]
+        raw = '\n'.join(lines)
+        run_humaneval_check(raw)

tests/dataset/test_local_datasets.py

@@ -0,0 +1,230 @@
+import random
+import sys
+import unittest
+import warnings
+from os import environ
+
+from datasets import Dataset, DatasetDict
+from mmengine.config import read_base
+from tqdm import tqdm
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+
+def reload_datasets():
+    modules_to_remove = [
+        module_name for module_name in sys.modules
+        if module_name.startswith('configs.datasets')
+    ]
+
+    for module_name in modules_to_remove:
+        del sys.modules[module_name]
+
+    with read_base():
+        from configs.datasets.ceval.ceval_gen import ceval_datasets
+        from configs.datasets.gsm8k.gsm8k_gen import gsm8k_datasets
+        from configs.datasets.cmmlu.cmmlu_gen import cmmlu_datasets
+        from configs.datasets.ARC_c.ARC_c_gen import ARC_c_datasets
+        from configs.datasets.ARC_e.ARC_e_gen import ARC_e_datasets
+        from configs.datasets.humaneval.humaneval_gen import humaneval_datasets
+        from configs.datasets.humaneval.humaneval_repeat10_gen_8e312c import humaneval_datasets as humaneval_repeat10_datasets
+        from configs.datasets.race.race_ppl import race_datasets
+        from configs.datasets.commonsenseqa.commonsenseqa_gen import commonsenseqa_datasets
+        
+        from configs.datasets.mmlu.mmlu_gen import mmlu_datasets
+        from configs.datasets.strategyqa.strategyqa_gen import strategyqa_datasets
+        from configs.datasets.bbh.bbh_gen import bbh_datasets
+        from configs.datasets.Xsum.Xsum_gen import Xsum_datasets
+        from configs.datasets.winogrande.winogrande_gen import winogrande_datasets
+        from configs.datasets.winogrande.winogrande_ll import winogrande_datasets as winogrande_ll_datasets
+        from configs.datasets.winogrande.winogrande_5shot_ll_252f01 import winogrande_datasets as winogrande_5shot_ll_datasets
+        from configs.datasets.obqa.obqa_gen import obqa_datasets
+        from configs.datasets.obqa.obqa_ppl_6aac9e import obqa_datasets as obqa_ppl_datasets
+        from configs.datasets.agieval.agieval_gen import agieval_datasets as agieval_v2_datasets
+        # from configs.datasets.agieval.agieval_gen_a0c741 import agieval_datasets as agieval_v1_datasets
+        from configs.datasets.siqa.siqa_gen import siqa_datasets as siqa_v2_datasets
+        from configs.datasets.siqa.siqa_gen_18632c import siqa_datasets as siqa_v3_datasets
+        from configs.datasets.siqa.siqa_ppl_42bc6e import siqa_datasets as siqa_ppl_datasets
+        from configs.datasets.storycloze.storycloze_gen import storycloze_datasets
+        from configs.datasets.storycloze.storycloze_ppl import storycloze_datasets as storycloze_ppl_datasets
+        from configs.datasets.summedits.summedits_gen import summedits_datasets as summedits_v2_datasets
+        
+        from configs.datasets.hellaswag.hellaswag_gen import hellaswag_datasets as hellaswag_v2_datasets
+        from configs.datasets.hellaswag.hellaswag_10shot_gen_e42710 import hellaswag_datasets as hellaswag_ice_datasets
+        from configs.datasets.hellaswag.hellaswag_ppl_9dbb12 import hellaswag_datasets as hellaswag_v1_datasets
+        from configs.datasets.hellaswag.hellaswag_ppl_a6e128 import hellaswag_datasets as hellaswag_v3_datasets
+        from configs.datasets.mbpp.mbpp_gen import mbpp_datasets as mbpp_v1_datasets
+        from configs.datasets.mbpp.mbpp_passk_gen_830460 import mbpp_datasets as mbpp_v2_datasets
+        from configs.datasets.mbpp.sanitized_mbpp_gen_830460 import sanitized_mbpp_datasets
+        from configs.datasets.nq.nq_gen import nq_datasets
+        from configs.datasets.lcsts.lcsts_gen import lcsts_datasets
+        from configs.datasets.math.math_gen import math_datasets
+        from configs.datasets.piqa.piqa_gen import piqa_datasets as piqa_v2_datasets
+        from configs.datasets.piqa.piqa_ppl import piqa_datasets as piqa_v1_datasets
+        from configs.datasets.piqa.piqa_ppl_0cfff2 import piqa_datasets as piqa_v3_datasets
+        from configs.datasets.lambada.lambada_gen import lambada_datasets
+        from configs.datasets.tydiqa.tydiqa_gen import tydiqa_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_gen import GaokaoBench_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_mixed import GaokaoBench_datasets as GaokaoBench_mixed_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_no_subjective_gen_4c31db import GaokaoBench_datasets as GaokaoBench_no_subjective_datasets
+        from configs.datasets.triviaqa.triviaqa_gen import triviaqa_datasets
+        from configs.datasets.triviaqa.triviaqa_wiki_1shot_gen_20a989 import triviaqa_datasets as triviaqa_wiki_1shot_datasets
+        
+        from configs.datasets.CLUE_cmnli.CLUE_cmnli_gen import cmnli_datasets
+        from configs.datasets.CLUE_cmnli.CLUE_cmnli_ppl import cmnli_datasets as cmnli_ppl_datasets
+        from configs.datasets.CLUE_ocnli.CLUE_ocnli_gen import ocnli_datasets
+        
+        from configs.datasets.ceval.ceval_clean_ppl import ceval_datasets as ceval_clean_datasets
+        from configs.datasets.ARC_c.ARC_c_clean_ppl import ARC_c_datasets as ARC_c_clean_datasets
+        from configs.datasets.mmlu.mmlu_clean_ppl import mmlu_datasets as mmlu_clean_datasets
+        from configs.datasets.hellaswag.hellaswag_clean_ppl import hellaswag_datasets as hellaswag_clean_datasets
+        from configs.datasets.FewCLUE_ocnli_fc.FewCLUE_ocnli_fc_gen import ocnli_fc_datasets
+
+    return sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+
+
+def load_datasets_conf(source):
+    environ['DATASET_SOURCE'] = source
+    datasets_conf = reload_datasets()
+    return datasets_conf
+
+
+def load_datasets(source, conf):
+    environ['DATASET_SOURCE'] = source
+    if 'lang' in conf:
+        dataset = conf['type'].load(path=conf['path'], lang=conf['lang'])
+        return dataset
+    if 'setting_name' in conf:
+        dataset = conf['type'].load(path=conf['path'],
+                                    name=conf['name'],
+                                    setting_name=conf['setting_name'])
+        return dataset
+    if 'name' in conf:
+        dataset = conf['type'].load(path=conf['path'], name=conf['name'])
+        return dataset
+
+    if 'local_mode' in conf:
+        dataset = conf['type'].load(path=conf['path'], local_mode=conf['local_mode'])
+        return dataset
+    try:
+        dataset = conf['type'].load(path=conf['path'])
+    except Exception:
+        dataset = conf['type'].load(**conf)
+    return dataset
+
+
+def clean_string(value):
+    """Helper function to clean and normalize string data.
+
+    It strips leading and trailing whitespace and replaces multiple whitespace
+    characters with a single space.
+    """
+    if isinstance(value, str):
+        return ' '.join(value.split())
+    return value
+
+
+class TestingLocalDatasets(unittest.TestCase):
+
+    def test_datasets(self):
+        # 加载 ModelScope 和 Local 数据集配置
+        # ms_datasets_conf = load_datasets_conf('ModelScope')
+        local_datasets_conf = load_datasets_conf('Local')
+        
+        # 初始化成功和失败的数据集列表
+        successful_comparisons = []
+        failed_comparisons = []
+        
+        def compare_datasets(local_conf):
+            # local_dataset = load_datasets(local_conf)
+            local_dataset = load_datasets('Local', local_conf)
+            # modelscope_path_name = f"{ms_conf.get('path')}/{ms_conf.get('name', '')}\t{ms_conf.get('lang', '')}"
+            local_path_name = f"{local_conf.get('path')}/{local_conf.get('name', '')}\t{local_conf.get('lang', '')}"
+            # # 断言类型一致
+            # assert ms_conf['type'] == local_conf['type'], "Data types do not match"
+            # print(modelscope_path_name, local_path_name)
+            try:
+            #     ms_dataset = load_datasets('ModelScope', ms_conf)
+                local_dataset = load_datasets('Local', local_conf)
+            #     _check_data(ms_dataset, local_dataset, sample_size=sample_size)
+                return 'success', f'{local_path_name}'
+            except Exception as exception:
+            #     print(exception)
+                return 'failure', f'can\'t load {local_path_name}'
+        
+        with ThreadPoolExecutor(16) as executor:
+            futures = {
+                executor.submit(compare_datasets, local_conf):  local_conf
+                for local_conf in  local_datasets_conf
+            }
+            
+            for future in tqdm(as_completed(futures), total=len(futures)):
+                result, message = future.result()
+                if result == 'success':
+                    successful_comparisons.append(message)
+                else:
+                    failed_comparisons.append(message)
+        
+        # 输出测试总结
+        total_datasets = len(local_datasets_conf)
+        print(f"All {total_datasets} datasets")
+        print(f"OK {len(successful_comparisons)} datasets")
+        for success in successful_comparisons:
+            print(f"  {success}")
+        print(f"Fail {len(failed_comparisons)} datasets")
+        for failure in failed_comparisons:
+            print(f"  {failure}")
+
+
+def _check_data(ms_dataset: Dataset | DatasetDict,
+                oc_dataset: Dataset | DatasetDict,
+                sample_size):
+    assert type(ms_dataset) == type(
+        oc_dataset
+    ), f'Dataset type not match: {type(ms_dataset)} != {type(oc_dataset)}'
+
+    # match DatasetDict
+    if isinstance(oc_dataset, DatasetDict):
+        assert ms_dataset.keys() == oc_dataset.keys(
+        ), f'DatasetDict not match: {ms_dataset.keys()} != {oc_dataset.keys()}'
+
+        for key in ms_dataset.keys():
+            _check_data(ms_dataset[key], oc_dataset[key], sample_size=sample_size)
+
+    elif isinstance(oc_dataset, Dataset):
+        # match by cols
+        assert set(ms_dataset.column_names) == set(
+            oc_dataset.column_names
+        ), f'Column names do not match: {ms_dataset.column_names} != {oc_dataset.column_names}'
+
+        # Check that the number of rows is the same
+        assert len(ms_dataset) == len(
+            oc_dataset
+        ), f'Number of rows do not match: {len(ms_dataset)} != {len(oc_dataset)}'
+
+        # Randomly sample indices
+        sample_indices = random.sample(range(len(ms_dataset)),
+                                       min(sample_size, len(ms_dataset)))
+
+        for i, idx in enumerate(sample_indices):
+            for col in ms_dataset.column_names:
+                ms_value = clean_string(str(ms_dataset[col][idx]))
+                oc_value = clean_string(str(oc_dataset[col][idx]))
+                try:
+                    assert ms_value == oc_value, f"Value mismatch in column '{col}', index {idx}: {ms_value} != {oc_value}"
+                except AssertionError as e:
+                    print(f"Assertion failed for column '{col}', index {idx}")
+                    print(f"ms_data: {ms_dataset[idx]}")
+                    print(f'oc_data: {oc_dataset[idx]}')
+                    print(f'ms_value: {ms_value} ({type(ms_value)})')
+                    print(f'oc_value: {oc_value} ({type(oc_value)})')
+                    raise e
+    else:
+        raise ValueError(f'Datasets type not supported {type(ms_dataset)}')
+
+
+if __name__ == '__main__':
+    sample_size = 100
+    unittest.main()

tests/dataset/test_ms_datasets.py

@@ -0,0 +1,226 @@
+import random
+import sys
+import unittest
+import warnings
+from os import environ
+
+from datasets import Dataset, DatasetDict
+from mmengine.config import read_base
+from tqdm import tqdm
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+
+def reload_datasets():
+    modules_to_remove = [
+        module_name for module_name in sys.modules
+        if module_name.startswith('configs.datasets')
+    ]
+
+    for module_name in modules_to_remove:
+        del sys.modules[module_name]
+
+    with read_base():
+        from configs.datasets.ceval.ceval_gen import ceval_datasets
+        from configs.datasets.gsm8k.gsm8k_gen import gsm8k_datasets
+        from configs.datasets.cmmlu.cmmlu_gen import cmmlu_datasets
+        from configs.datasets.ARC_c.ARC_c_gen import ARC_c_datasets
+        from configs.datasets.ARC_e.ARC_e_gen import ARC_e_datasets
+        from configs.datasets.humaneval.humaneval_gen import humaneval_datasets
+        from configs.datasets.humaneval.humaneval_repeat10_gen_8e312c import humaneval_datasets as humaneval_repeat10_datasets
+        from configs.datasets.race.race_ppl import race_datasets
+        from configs.datasets.commonsenseqa.commonsenseqa_gen import commonsenseqa_datasets
+        
+        from configs.datasets.mmlu.mmlu_gen import mmlu_datasets
+        from configs.datasets.bbh.bbh_gen import bbh_datasets
+        from configs.datasets.Xsum.Xsum_gen import Xsum_datasets
+        from configs.datasets.winogrande.winogrande_gen import winogrande_datasets
+        from configs.datasets.winogrande.winogrande_ll import winogrande_datasets as winogrande_ll_datasets
+        from configs.datasets.winogrande.winogrande_5shot_ll_252f01 import winogrande_datasets as winogrande_5shot_ll_datasets
+        from configs.datasets.obqa.obqa_gen import obqa_datasets
+        from configs.datasets.obqa.obqa_ppl_6aac9e import obqa_datasets as obqa_ppl_datasets
+        from configs.datasets.agieval.agieval_gen import agieval_datasets as agieval_v2_datasets
+        from configs.datasets.agieval.agieval_gen_a0c741 import agieval_datasets as agieval_v1_datasets
+        from configs.datasets.siqa.siqa_gen import siqa_datasets as siqa_v2_datasets
+        from configs.datasets.siqa.siqa_gen_18632c import siqa_datasets as siqa_v3_datasets
+        from configs.datasets.siqa.siqa_ppl_42bc6e import siqa_datasets as siqa_ppl_datasets
+        from configs.datasets.storycloze.storycloze_gen import storycloze_datasets
+        from configs.datasets.storycloze.storycloze_ppl import storycloze_datasets as storycloze_ppl_datasets
+        from configs.datasets.summedits.summedits_gen import summedits_datasets as summedits_v2_datasets
+        
+        from configs.datasets.strategyqa.strategyqa_gen import strategyqa_datasets
+        from configs.datasets.mbpp.mbpp_gen import mbpp_datasets as mbpp_v1_datasets
+        from configs.datasets.lcsts.lcsts_gen import lcsts_datasets
+        
+        from configs.datasets.hellaswag.hellaswag_gen import hellaswag_datasets as hellaswag_v2_datasets
+        from configs.datasets.hellaswag.hellaswag_10shot_gen_e42710 import hellaswag_datasets as hellaswag_ice_datasets
+        from configs.datasets.hellaswag.hellaswag_ppl_9dbb12 import hellaswag_datasets as hellaswag_v1_datasets
+        from configs.datasets.hellaswag.hellaswag_ppl_a6e128 import hellaswag_datasets as hellaswag_v3_datasets
+        from configs.datasets.mbpp.mbpp_passk_gen_830460 import mbpp_datasets as mbpp_v2_datasets
+        from configs.datasets.mbpp.sanitized_mbpp_gen_830460 import sanitized_mbpp_datasets
+        from configs.datasets.nq.nq_gen import nq_datasets
+        from configs.datasets.math.math_gen import math_datasets
+        from configs.datasets.piqa.piqa_gen import piqa_datasets as piqa_v2_datasets
+        from configs.datasets.piqa.piqa_ppl import piqa_datasets as piqa_v1_datasets
+        from configs.datasets.piqa.piqa_ppl_0cfff2 import piqa_datasets as piqa_v3_datasets
+        from configs.datasets.lambada.lambada_gen import lambada_datasets
+        from configs.datasets.tydiqa.tydiqa_gen import tydiqa_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_gen import GaokaoBench_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_mixed import GaokaoBench_datasets as GaokaoBench_mixed_datasets
+        from configs.datasets.GaokaoBench.GaokaoBench_no_subjective_gen_4c31db import GaokaoBench_datasets as GaokaoBench_no_subjective_datasets
+        from configs.datasets.triviaqa.triviaqa_gen import triviaqa_datasets
+        from configs.datasets.triviaqa.triviaqa_wiki_1shot_gen_20a989 import triviaqa_datasets as triviaqa_wiki_1shot_datasets
+        
+        from configs.datasets.CLUE_cmnli.CLUE_cmnli_gen import cmnli_datasets
+        from configs.datasets.CLUE_cmnli.CLUE_cmnli_ppl import cmnli_datasets as cmnli_ppl_datasets
+        from configs.datasets.CLUE_ocnli.CLUE_ocnli_gen import ocnli_datasets
+        
+        from configs.datasets.ceval.ceval_clean_ppl import ceval_datasets as ceval_clean_datasets
+        from configs.datasets.ARC_c.ARC_c_clean_ppl import ARC_c_datasets as ARC_c_clean_datasets
+        from configs.datasets.mmlu.mmlu_clean_ppl import mmlu_datasets as mmlu_clean_datasets
+        from configs.datasets.hellaswag.hellaswag_clean_ppl import hellaswag_datasets as hellaswag_clean_datasets
+        
+    return sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+
+
+def load_datasets_conf(source):
+    environ['DATASET_SOURCE'] = source
+    datasets_conf = reload_datasets()
+    return datasets_conf
+
+
+def load_datasets(source, conf):
+    environ['DATASET_SOURCE'] = source
+    if 'lang' in conf:
+        dataset = conf['type'].load(path=conf['path'], lang=conf['lang'])
+        return dataset
+    if 'setting_name' in conf:
+        dataset = conf['type'].load(path=conf['path'],
+                                    name=conf['name'],
+                                    setting_name=conf['setting_name'])
+        return dataset
+    if 'name' in conf:
+        dataset = conf['type'].load(path=conf['path'], name=conf['name'])
+        return dataset
+    try:
+        dataset = conf['type'].load(path=conf['path'])
+    except Exception as e:
+        print(e)
+        dataset = conf['type'].load(**conf)
+    return dataset
+
+
+def clean_string(value):
+    """Helper function to clean and normalize string data.
+
+    It strips leading and trailing whitespace and replaces multiple whitespace
+    characters with a single space.
+    """
+    if isinstance(value, str):
+        return ' '.join(value.split())
+    return value
+
+
+class TestingMsDatasets(unittest.TestCase):
+
+    def test_datasets(self):
+        # 加载 ModelScope 和 Local 数据集配置
+        ms_datasets_conf = load_datasets_conf('ModelScope')
+        local_datasets_conf = load_datasets_conf('Local')
+        
+        # 初始化成功和失败的数据集列表
+        successful_comparisons = []
+        failed_comparisons = []
+        
+        def compare_datasets(ms_conf, local_conf):
+            modelscope_path_name = f"{ms_conf.get('path')}/{ms_conf.get('name', '')}\t{ms_conf.get('lang', '')}"
+            local_path_name = f"{local_conf.get('path')}/{local_conf.get('name', '')}\t{local_conf.get('lang', '')}"
+            # 断言类型一致
+            assert ms_conf['type'] == local_conf['type'], "Data types do not match"
+            print(modelscope_path_name, local_path_name)
+            try:
+                ms_dataset = load_datasets('ModelScope', ms_conf)
+                local_dataset = load_datasets('Local', local_conf)
+                _check_data(ms_dataset, local_dataset, sample_size=sample_size)
+                return 'success', f'{modelscope_path_name} | {local_path_name}'
+            except Exception as exception:
+                print(exception)
+                return 'failure', f'{modelscope_path_name} is not the same as {local_path_name}'
+        
+        with ThreadPoolExecutor(thread) as executor:
+            futures = {
+                executor.submit(compare_datasets, ms_conf, local_conf): (ms_conf, local_conf)
+                for ms_conf, local_conf in zip(ms_datasets_conf, local_datasets_conf)
+            }
+            
+            for future in tqdm(as_completed(futures), total=len(futures)):
+                result, message = future.result()
+                if result == 'success':
+                    successful_comparisons.append(message)
+                else:
+                    failed_comparisons.append(message)
+        
+        # 输出测试总结
+        total_datasets = len(ms_datasets_conf)
+        print(f"All {total_datasets} datasets")
+        print(f"OK {len(successful_comparisons)} datasets")
+        for success in successful_comparisons:
+            print(f"  {success}")
+        print(f"Fail {len(failed_comparisons)} datasets")
+        for failure in failed_comparisons:
+            print(f"  {failure}")
+
+
+def _check_data(ms_dataset: Dataset | DatasetDict,
+                oc_dataset: Dataset | DatasetDict,
+                sample_size):
+    assert type(ms_dataset) == type(
+        oc_dataset
+    ), f'Dataset type not match: {type(ms_dataset)} != {type(oc_dataset)}'
+
+    # match DatasetDict
+    if isinstance(oc_dataset, DatasetDict):
+        assert ms_dataset.keys() == oc_dataset.keys(
+        ), f'DatasetDict not match: {ms_dataset.keys()} != {oc_dataset.keys()}'
+
+        for key in ms_dataset.keys():
+            _check_data(ms_dataset[key], oc_dataset[key], sample_size=sample_size)
+
+    elif isinstance(oc_dataset, Dataset):
+        # match by cols
+        assert set(ms_dataset.column_names) == set(
+            oc_dataset.column_names
+        ), f'Column names do not match: {ms_dataset.column_names} != {oc_dataset.column_names}'
+
+        # Check that the number of rows is the same
+        assert len(ms_dataset) == len(
+            oc_dataset
+        ), f'Number of rows do not match: {len(ms_dataset)} != {len(oc_dataset)}'
+
+        # Randomly sample indices
+        sample_indices = random.sample(range(len(ms_dataset)),
+                                       min(sample_size, len(ms_dataset)))
+
+        for i, idx in enumerate(sample_indices):
+            for col in ms_dataset.column_names:
+                ms_value = clean_string(str(ms_dataset[col][idx]))
+                oc_value = clean_string(str(oc_dataset[col][idx]))
+                try:
+                    assert ms_value == oc_value, f"Value mismatch in column '{col}', index {idx}: {ms_value} != {oc_value}"
+                except AssertionError as e:
+                    print(f"Assertion failed for column '{col}', index {idx}")
+                    print(f"ms_data: {ms_dataset[idx]}")
+                    print(f'oc_data: {oc_dataset[idx]}')
+                    print(f'ms_value: {ms_value} ({type(ms_value)})')
+                    print(f'oc_value: {oc_value} ({type(oc_value)})')
+                    raise e
+    else:
+        raise ValueError(f'Datasets type not supported {type(ms_dataset)}')
+
+
+if __name__ == '__main__':
+    sample_size = 100
+    thread = 1
+    unittest.main()

tests/openicl/test_prompt_template.py

@@ -0,0 +1,249 @@
+import unittest
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.utils.prompt import PromptList
+
+
+class TestPromptTemplate(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.qa_template = dict(begin=[
+            dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'),
+            '</E>',
+        ],
+                                round=[
+                                    dict(role='HUMAN', prompt='{input}'),
+                                    dict(role='BOT', prompt='Answer: {answer}')
+                                ])
+        self.multiround_qa_template = dict(round=[
+            dict(role='HUMAN', prompt='{input}'),
+            dict(role='BOT', prompt='A1', end='\n'),
+            dict(role='HUMAN', prompt='Q1'),
+            dict(role='BOT', prompt='A2', end='\n\n'),
+            dict(role='HUMAN', prompt='Q2', begin='HUMAN:'),
+            dict(role='BOT', prompt='Answer: {answer}')
+        ])
+        self.entry = {'input': 'Hello, how are you?', 'answer': 'Good.'}
+
+    def test_init(self):
+        template = 'Translate the following English text to French: {input}.'
+        pt = PromptTemplate(template)
+
+        self.assertEqual(pt.template, template)
+
+    def test_generate_ice_item(self):
+        # Test simple prompt
+        template = 'Translate the following English text to French: {input}.'
+        pt = PromptTemplate(template)
+        label = None
+        ice = pt.generate_ice_item(self.entry, label)
+
+        self.assertEqual(ice,
+                         ('Translate the following English text to French: '
+                          'Hello, how are you?.'))
+
+        # test meta prompt style
+        pt = PromptTemplate(self.qa_template, ice_token='</E>')
+        label = None
+        ice = pt.generate_ice_item(self.entry, label)
+
+        ice_target = PromptList([
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(ice, ice_target)
+
+        # test_multiround
+        pt = PromptTemplate(self.multiround_qa_template, ice_token='</E>')
+        label = None
+        ice = pt.generate_ice_item(self.entry, label)
+
+        ice_target = PromptList([
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='A1', end='\n'),
+            dict(role='HUMAN', prompt='Q1'),
+            dict(role='BOT', prompt='A2', end='\n\n'),
+            dict(role='HUMAN', prompt='Q2', begin='HUMAN:'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(ice, ice_target)
+
+    def test_generate_label_prompt_item(self):
+        # Test simple prompt
+        template = ('</E> Translate the following English text to French: '
+                    '{input}.')
+        pt = PromptTemplate(template, ice_token='</E>')
+        ice = 'ICE'
+        label = None
+        prompt = pt.generate_label_prompt_item(self.entry, ice, label)
+
+        self.assertEqual(
+            prompt, ('ICE Translate the following English text to French: '
+                     'Hello, how are you?.'))
+
+        ice = PromptList([
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='h1'),
+            dict(role='BOT', prompt='b1'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+        ])
+
+        # test meta prompt style
+        pt = PromptTemplate(self.qa_template, ice_token='</E>')
+        label = None
+        prompt = pt.generate_label_prompt_item(self.entry, ice, label)
+        target = PromptList([
+            {
+                'section': 'begin',
+                'pos': 'begin'
+            },
+            dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'),
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='h1'),
+            dict(role='BOT', prompt='b1'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+            {
+                'section': 'begin',
+                'pos': 'end'
+            },
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(prompt, target)
+
+        # test_multiround
+        pt = PromptTemplate(self.multiround_qa_template, ice_token='</E>')
+        label = None
+        prompt = pt.generate_label_prompt_item(self.entry, ice, label)
+        target = PromptList([
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='A1', end='\n'),
+            dict(role='HUMAN', prompt='Q1'),
+            dict(role='BOT', prompt='A2', end='\n\n'),
+            dict(role='HUMAN', prompt='Q2', begin='HUMAN:'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(prompt, target)
+
+    def test_generate_item(self):
+        # Test simple prompt
+        template = 'Translate the following English text to French: {input}.'
+        pt = PromptTemplate(template)
+        item = pt.generate_item(self.entry)
+
+        self.assertEqual(item,
+                         ('Translate the following English text to French: '
+                          'Hello, how are you?.'))
+
+        ice = PromptList([
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='h1'),
+            dict(role='BOT', prompt='b1'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+        ])
+
+        # test meta prompt (without system role)
+        pt = PromptTemplate(self.qa_template, ice_token='</E>')
+        prompt = pt.generate_item(self.entry, ice_field_replace_token=ice)
+        target = PromptList([
+            {
+                'section': 'begin',
+                'pos': 'begin'
+            },
+            dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'),
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='h1'),
+            dict(role='BOT', prompt='b1'),
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+            {
+                'section': 'begin',
+                'pos': 'end'
+            },
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(prompt, target)
+
+        pt = PromptTemplate(self.multiround_qa_template, ice_token='</E>')
+        prompt = pt.generate_item(self.entry, ice_field_replace_token=ice)
+        target = PromptList([
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            dict(role='HUMAN', prompt='Hello, how are you?'),
+            dict(role='BOT', prompt='A1', end='\n'),
+            dict(role='HUMAN', prompt='Q1'),
+            dict(role='BOT', prompt='A2', end='\n\n'),
+            dict(role='HUMAN', prompt='Q2', begin='HUMAN:'),
+            dict(role='BOT', prompt='Answer: Good.'),
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+        ])
+        self.assertEqual(prompt, target)

tests/prompt/test_api_template_parser.py

@@ -0,0 +1,231 @@
+import unittest
+
+from opencompass.models.base_api import APITemplateParser
+from opencompass.utils.prompt import PromptList
+
+
+class TestAPITemplateParser(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = APITemplateParser()
+        self.prompt = PromptList([
+            {
+                'section': 'begin',
+                'pos': 'begin'
+            },
+            'begin',
+            {
+                'role': 'SYSTEM',
+                'fallback_role': 'HUMAN',
+                'prompt': 'system msg'
+            },
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U0'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B0'
+            },
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+            {
+                'section': 'begin',
+                'pos': 'end'
+            },
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U1'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B1'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U2'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B2'
+            },
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+            {
+                'section': 'end',
+                'pos': 'begin'
+            },
+            'end',
+            {
+                'section': 'end',
+                'pos': 'end'
+            },
+        ])
+
+    def test_parse_template_str_input(self):
+        prompt = self.parser.parse_template('Hello, world!', mode='gen')
+        self.assertEqual(prompt, 'Hello, world!')
+        prompt = self.parser.parse_template('Hello, world!', mode='ppl')
+        self.assertEqual(prompt, 'Hello, world!')
+
+    def test_parse_template_list_input(self):
+        prompt = self.parser.parse_template(['Hello', 'world'], mode='gen')
+        self.assertEqual(prompt, ['Hello', 'world'])
+        prompt = self.parser.parse_template(['Hello', 'world'], mode='ppl')
+        self.assertEqual(prompt, ['Hello', 'world'])
+
+    def test_parse_template_PromptList_input_no_meta_template(self):
+        prompt = self.parser.parse_template(self.prompt, mode='gen')
+        self.assertEqual(prompt,
+                         'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend')
+        prompt = self.parser.parse_template(self.prompt, mode='ppl')
+        self.assertEqual(prompt,
+                         'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend')
+
+    def test_parse_template_PromptList_input_with_meta_template(self):
+        parser = APITemplateParser(meta_template=dict(round=[
+            dict(role='HUMAN', api_role='HUMAN'),
+            dict(role='BOT', api_role='BOT', generate=True)
+        ], ))
+        with self.assertWarns(Warning):
+            prompt = parser.parse_template(self.prompt, mode='gen')
+            self.assertEqual(
+                prompt,
+                PromptList([
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'system msg\nU0'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B0'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U1'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B1'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U2'
+                    },
+                ]))
+        with self.assertWarns(Warning):
+            prompt = parser.parse_template(self.prompt, mode='ppl')
+            self.assertEqual(
+                prompt,
+                PromptList([
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'system msg\nU0'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B0'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U1'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B1'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U2'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B2'
+                    },
+                ]))
+
+        parser = APITemplateParser(meta_template=dict(
+            round=[
+                dict(role='HUMAN', api_role='HUMAN'),
+                dict(role='BOT', api_role='BOT', generate=True)
+            ],
+            reserved_roles=[
+                dict(role='SYSTEM', api_role='SYSTEM'),
+            ],
+        ))
+        with self.assertWarns(Warning):
+            prompt = parser.parse_template(self.prompt, mode='gen')
+            self.assertEqual(
+                prompt,
+                PromptList([
+                    {
+                        'role': 'SYSTEM',
+                        'prompt': 'system msg'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U0'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B0'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U1'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B1'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U2'
+                    },
+                ]))
+        with self.assertWarns(Warning):
+            prompt = parser.parse_template(self.prompt, mode='ppl')
+            self.assertEqual(
+                prompt,
+                PromptList([
+                    {
+                        'role': 'SYSTEM',
+                        'prompt': 'system msg'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U0'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B0'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U1'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B1'
+                    },
+                    {
+                        'role': 'HUMAN',
+                        'prompt': 'U2'
+                    },
+                    {
+                        'role': 'BOT',
+                        'prompt': 'B2'
+                    },
+                ]))

tests/prompt/test_lm_template_parser.py

@@ -0,0 +1,235 @@
+import unittest
+
+from opencompass.models.base import LMTemplateParser
+from opencompass.utils.prompt import PromptList
+
+
+class TestLMTemplateParser(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = LMTemplateParser()
+        self.prompt = PromptList([
+            {
+                'section': 'begin',
+                'pos': 'begin'
+            },
+            'begin',
+            {
+                'role': 'SYSTEM',
+                'fallback_role': 'HUMAN',
+                'prompt': 'system msg'
+            },
+            {
+                'section': 'ice',
+                'pos': 'begin'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U0'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B0'
+            },
+            {
+                'section': 'ice',
+                'pos': 'end'
+            },
+            {
+                'section': 'begin',
+                'pos': 'end'
+            },
+            {
+                'section': 'round',
+                'pos': 'begin'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U1',
+                'end': '\n'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B1'
+            },
+            {
+                'role': 'HUMAN',
+                'prompt': 'U2'
+            },
+            {
+                'role': 'BOT',
+                'prompt': 'B2'
+            },
+            {
+                'section': 'round',
+                'pos': 'end'
+            },
+            {
+                'section': 'end',
+                'pos': 'begin'
+            },
+            'end',
+            {
+                'section': 'end',
+                'pos': 'end'
+            },
+        ])
+
+    def test_parse_template_str_input(self):
+        prompt = self.parser.parse_template('Hello, world!', mode='gen')
+        self.assertEqual(prompt, 'Hello, world!')
+        prompt = self.parser.parse_template('Hello, world!', mode='ppl')
+        self.assertEqual(prompt, 'Hello, world!')
+
+    def test_parse_template_list_input(self):
+        prompt = self.parser.parse_template(['Hello', 'world'], mode='gen')
+        self.assertEqual(prompt, ['Hello', 'world'])
+        prompt = self.parser.parse_template(['Hello', 'world'], mode='ppl')
+        self.assertEqual(prompt, ['Hello', 'world'])
+
+    def test_parse_template_PromptList_input_no_meta_template(self):
+        prompt = self.parser.parse_template(self.prompt, mode='gen')
+        self.assertEqual(prompt,
+                         'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend')
+        prompt = self.parser.parse_template(self.prompt, mode='ppl')
+        self.assertEqual(prompt,
+                         'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend')
+
+    def test_parse_template_PromptList_input_with_meta_template(self):
+        # no SYSTEM role, early generation in THOUGHTS
+        parser = LMTemplateParser(meta_template=dict(
+            begin='meta instruction\n',
+            round=[
+                dict(role='HUMAN', begin='<|HUMAN|>:', end='<eoh>\n'),
+                dict(role='THOUGHTS',
+                     begin='<|Inner Thoughts|>:',
+                     generate=True,
+                     end='<eot>\n',
+                     prompt='None'),
+                dict(role='BOT', begin='<|BOT|>:', end='<eob>\n'),
+            ],
+            end='meta end',
+        ))
+        prompt = parser.parse_template(self.prompt, mode='gen')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|HUMAN|>:system msg<eoh>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:')
+        self.assertEqual(prompt, target)
+        prompt = parser.parse_template(self.prompt, mode='ppl')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|HUMAN|>:system msg<eoh>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B2<eob>\n'
+                  'end'
+                  'meta end')
+        self.assertEqual(prompt, target)
+
+        # no SYSTEM role, generation in BOT
+        parser = LMTemplateParser(meta_template=dict(
+            begin='meta instruction\n',
+            round=[
+                dict(role='HUMAN', begin='<|HUMAN|>:', end='<eoh>\n'),
+                dict(role='THOUGHTS',
+                     begin='<|Inner Thoughts|>:',
+                     end='<eot>\n',
+                     prompt='None'),
+                dict(
+                    role='BOT', begin='<|BOT|>:', end='<eob>\n',
+                    generate=True),
+            ],
+            end='meta end',
+        ))
+        prompt = parser.parse_template(self.prompt, mode='gen')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|HUMAN|>:system msg<eoh>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:')
+        self.assertEqual(prompt, target)
+        prompt = parser.parse_template(self.prompt, mode='ppl')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|HUMAN|>:system msg<eoh>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B2<eob>\n'
+                  'end'
+                  'meta end')
+        self.assertEqual(prompt, target)
+
+        # with SYSTEM role, generation in BOT
+        parser = LMTemplateParser(meta_template=dict(
+            begin='meta instruction\n',
+            round=[
+                dict(role='HUMAN', begin='<|HUMAN|>:', end='<eoh>\n'),
+                dict(role='THOUGHTS',
+                     begin='<|Inner Thoughts|>:',
+                     end='<eot>\n',
+                     prompt='None'),
+                dict(
+                    role='BOT', begin='<|BOT|>:', end='<eob>\n',
+                    generate=True),
+            ],
+            end='meta end',
+            reserved_roles=[
+                dict(role='SYSTEM', begin='<|SYSTEM|>:', end='<eos>\n')
+            ]))
+        prompt = parser.parse_template(self.prompt, mode='gen')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|SYSTEM|>:system msg<eos>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:')
+        self.assertEqual(prompt, target)
+        prompt = parser.parse_template(self.prompt, mode='ppl')
+        target = ('meta instruction\n'
+                  'begin'
+                  '<|SYSTEM|>:system msg<eos>\n'
+                  '<|HUMAN|>:U0<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B0<eob>\n'
+                  '<|HUMAN|>:U1\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B1<eob>\n'
+                  '<|HUMAN|>:U2<eoh>\n'
+                  '<|Inner Thoughts|>:None<eot>\n'
+                  '<|BOT|>:B2<eob>\n'
+                  'end'
+                  'meta end')
+        self.assertEqual(prompt, target)

tests/prompt/test_prompt_list.py

@@ -0,0 +1,76 @@
+import unittest
+
+from opencompass.utils.prompt import PromptList
+
+
+class TestPromptList(unittest.TestCase):
+
+    def test_initialization(self):
+        pl = PromptList()
+        self.assertEqual(pl, [])
+
+        pl = PromptList(['test', '123'])
+        self.assertEqual(pl, ['test', '123'])
+
+    def test_format(self):
+        pl = PromptList(['hi {a}{b}', {'prompt': 'hey {a}!'}, '123'])
+        new_pl = pl.format(a=1, b=2, c=3)
+        self.assertEqual(new_pl, ['hi 12', {'prompt': 'hey 1!'}, '123'])
+
+        new_pl = pl.format(b=2)
+        self.assertEqual(new_pl, ['hi {a}2', {'prompt': 'hey {a}!'}, '123'])
+
+        new_pl = pl.format(d=1)
+        self.assertEqual(new_pl, ['hi {a}{b}', {'prompt': 'hey {a}!'}, '123'])
+
+    def test_replace(self):
+        pl = PromptList(['hello world', {'prompt': 'hello world'}, '123'])
+        new_pl = pl.replace('world', 'there')
+        self.assertEqual(new_pl,
+                         ['hello there', {
+                             'prompt': 'hello there'
+                         }, '123'])
+
+        new_pl = pl.replace('123', PromptList(['p', {'role': 'BOT'}]))
+        self.assertEqual(
+            new_pl,
+            ['hello world', {
+                'prompt': 'hello world'
+            }, 'p', {
+                'role': 'BOT'
+            }])
+
+        new_pl = pl.replace('2', PromptList(['p', {'role': 'BOT'}]))
+        self.assertEqual(new_pl, [
+            'hello world', {
+                'prompt': 'hello world'
+            }, '1', 'p', {
+                'role': 'BOT'
+            }, '3'
+        ])
+
+        with self.assertRaises(TypeError):
+            new_pl = pl.replace('world', PromptList(['new', 'world']))
+
+    def test_add(self):
+        pl = PromptList(['hello'])
+        new_pl = pl + ' world'
+        self.assertEqual(new_pl, ['hello', ' world'])
+
+        pl2 = PromptList([' world'])
+        new_pl = pl + pl2
+        self.assertEqual(new_pl, ['hello', ' world'])
+
+        new_pl = 'hi, ' + pl
+        self.assertEqual(new_pl, ['hi, ', 'hello'])
+
+        pl += '!'
+        self.assertEqual(pl, ['hello', '!'])
+
+    def test_str(self):
+        pl = PromptList(['hello', ' world', {'prompt': '!'}])
+        self.assertEqual(str(pl), 'hello world!')
+
+        with self.assertRaises(TypeError):
+            pl = PromptList(['hello', ' world', 123])
+            str(pl)