File size: 1,474 Bytes
76d1db9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | # Chat-aware MBPP for LLaDA-8B-Instruct. Stock mbpp.yaml has NO filter at all -- the
# raw response string is executed -- so a chat answer ("Here's the code: ```python...")
# fails on prose exactly like humaneval does. Added here:
# filter_fn: extract_code_mbpp (falls back to the raw string when unfenced)
# until: keeps "[DONE]", which the 3-shot exemplars teach the model to emit,
# and adds <|eot_id|> for the turns where it answers in chat style.
# Everything else, including the fewshot exemplars, is the stock task.
task: mbpp_instruct
dataset_path: google-research-datasets/mbpp
dataset_name: full
unsafe_code: true
output_type: generate_until
test_split: test
doc_to_text: "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n"
doc_to_target: "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}"
target_delimiter: ""
metric_list:
- metric: !function utils.pass_at_1
aggregation: mean
higher_is_better: true
generation_kwargs:
until:
- "[DONE]"
- "<|eot_id|>"
do_sample: false
num_fewshot: 3
fewshot_config:
sampler: first_n
samples: !function utils.list_fewshot_samples
filter_list:
- name: "extract_code"
filter:
- function: "custom"
filter_fn: !function utils.extract_code_mbpp
metadata:
version: 1.0
|