File size: 677 Bytes
b0c0df0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
dataset_path: neulab/VisualPuzzles
dataset_kwargs:
token: True
task: "VisualPuzzles_cot"
test_split: train
output_type: generate_until
doc_to_visual: !function utils.VisualPuzzles_doc_to_visual
doc_to_text: !function utils.VisualPuzzles_doc_to_text
doc_to_target: "answer"
generation_kwargs:
max_new_tokens: 4096
temperature: 0
top_p: 1.0
num_beams: 1
do_sample: false
metric_list:
- metric: exact_match
aggregation: mean
higher_is_better: true
ignore_case: true
ignore_punctuation: true
process_results: !function utils.VisualPuzzles_process_result
metadata:
- version: 0.0
lmms_eval_specific_kwargs:
default:
prompt: "COT_PROMPT"
|