| datasets = [ | |
| [ | |
| dict( | |
| abbr='ceval-computer_network', | |
| eval_cfg=dict( | |
| evaluator=dict( | |
| type='opencompass.openicl.icl_evaluator.AccEvaluator')), | |
| infer_cfg=dict( | |
| ice_template=dict( | |
| ice_token='</E>', | |
| template=dict( | |
| A=dict( | |
| begin='</E>', | |
| round=[ | |
| dict( | |
| prompt= | |
| '以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ', | |
| role='HUMAN'), | |
| dict(prompt='A', role='BOT'), | |
| ]), | |
| B=dict( | |
| begin='</E>', | |
| round=[ | |
| dict( | |
| prompt= | |
| '以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ', | |
| role='HUMAN'), | |
| dict(prompt='B', role='BOT'), | |
| ]), | |
| C=dict( | |
| begin='</E>', | |
| round=[ | |
| dict( | |
| prompt= | |
| '以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ', | |
| role='HUMAN'), | |
| dict(prompt='C', role='BOT'), | |
| ]), | |
| D=dict( | |
| begin='</E>', | |
| round=[ | |
| dict( | |
| prompt= | |
| '以下是中国关于计算机网络考试的单项选择题，请选出其中的正确答案。\n{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: ', | |
| role='HUMAN'), | |
| dict(prompt='D', role='BOT'), | |
| ])), | |
| type= | |
| 'opencompass.openicl.icl_prompt_template.PromptTemplate'), | |
| inferencer=dict( | |
| type='opencompass.openicl.icl_inferencer.PPLInferencer'), | |
| retriever=dict( | |
| fix_id_list=[ | |
| 0, | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| ], | |
| type='opencompass.openicl.icl_retriever.FixKRetriever')), | |
| name='computer_network', | |
| path='./data/ceval/formal_ceval', | |
| reader_cfg=dict( | |
| input_columns=[ | |
| 'question', | |
| 'A', | |
| 'B', | |
| 'C', | |
| 'D', | |
| ], | |
| output_column='answer', | |
| test_split='val', | |
| train_split='dev'), | |
| type='opencompass.datasets.CEvalDataset'), | |
| ], | |
| ] | |
| eval = dict(runner=dict(task=dict())) | |
| models = [ | |
| dict( | |
| abbr='llama-7b-hf', | |
| batch_padding=False, | |
| batch_size=8, | |
| max_out_len=100, | |
| max_seq_len=2048, | |
| model_kwargs=dict(device_map='auto'), | |
| path='huggyllama/llama-7b', | |
| run_cfg=dict(num_gpus=1, num_procs=1), | |
| tokenizer_kwargs=dict( | |
| padding_side='left', truncation_side='left', use_fast=False), | |
| tokenizer_path='huggyllama/llama-7b', | |
| type='opencompass.models.HuggingFaceCausalLM'), | |
| ] | |
| work_dir = './outputs/default/20240304_175403' | |