File size: 681 Bytes
3a464db | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | task: humaneval
dataset_path: openai/openai_humaneval
unsafe_code: true
output_type: generate_until
test_split: test
doc_to_text: "{{prompt}}"
doc_to_target: "{{test}}\ncheck({{entry_point}})"
process_docs: !function utils.process_docs
metric_list:
- metric: !function utils.pass_at_1
aggregation: mean
higher_is_better: true
generation_kwargs:
until:
- "\nclass"
- "\ndef"
- "\n#"
- "\nif"
- "\nprint"
- "\n```\n"
max_gen_toks: 2048
do_sample: false
repeats: 1
num_fewshot: 0
# filter_list:
# - name: "create_test"
# filter:
# - function: "custom"
# filter_fn: !function utils.build_predictions
metadata:
version: 1.0
|