| { | |
| "results": { | |
| "gsm8k": { | |
| "name": "gsm8k", | |
| "alias": "gsm8k", | |
| "sample_len": 20, | |
| "exact_match,strict-match": 0.65, | |
| "exact_match_stderr,strict-match": 0.1094243309804831, | |
| "exact_match,flexible-extract": 0.65, | |
| "exact_match_stderr,flexible-extract": 0.1094243309804831 | |
| }, | |
| "mmlu_high_school_mathematics_generative": { | |
| "name": "mmlu_high_school_mathematics_generative", | |
| "alias": "high_school_mathematics", | |
| "sample_len": 20, | |
| "exact_match,get_response": 0.0, | |
| "exact_match_stderr,get_response": 0.0 | |
| }, | |
| "mmlu_high_school_computer_science_generative": { | |
| "name": "mmlu_high_school_computer_science_generative", | |
| "alias": "high_school_computer_science", | |
| "sample_len": 20, | |
| "exact_match,get_response": 0.0, | |
| "exact_match_stderr,get_response": 0.0 | |
| }, | |
| "ifeval": { | |
| "name": "ifeval", | |
| "alias": "ifeval", | |
| "sample_len": 20, | |
| "prompt_level_strict_acc,none": 0.5, | |
| "prompt_level_strict_acc_stderr,none": 0.11470786693528086, | |
| "inst_level_strict_acc,none": 0.6, | |
| "inst_level_strict_acc_stderr,none": "N/A", | |
| "prompt_level_loose_acc,none": 0.5, | |
| "prompt_level_loose_acc_stderr,none": 0.11470786693528086, | |
| "inst_level_loose_acc,none": 0.6, | |
| "inst_level_loose_acc_stderr,none": "N/A" | |
| } | |
| }, | |
| "configs": { | |
| "gsm8k": { | |
| "task": "gsm8k", | |
| "dataset_path": "openai/gsm8k", | |
| "dataset_name": "main", | |
| "training_split": "train", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "doc_to_target": "{{answer}}", | |
| "unsafe_code": false, | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "default", | |
| "split": "train", | |
| "process_docs": null, | |
| "fewshot_indices": null, | |
| "samples": null, | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "doc_to_choice": null, | |
| "doc_to_target": "{{answer}}", | |
| "gen_prefix": null, | |
| "fewshot_delimiter": "\n\n", | |
| "target_delimiter": " " | |
| }, | |
| "num_fewshot": 5, | |
| "metric_list": [ | |
| { | |
| "metric": "exact_match", | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "ignore_case": true, | |
| "ignore_punctuation": false, | |
| "regexes_to_ignore": [ | |
| ",", | |
| "\\$", | |
| "(?s).*#### ", | |
| "\\.$" | |
| ] | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [ | |
| "Question:", | |
| "</s>", | |
| "<|im_end|>" | |
| ], | |
| "do_sample": false, | |
| "temperature": 0.0 | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "strict-match", | |
| "filter": [ | |
| { | |
| "function": "regex", | |
| "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "flexible-extract", | |
| "filter": [ | |
| { | |
| "function": "regex", | |
| "group_select": -1, | |
| "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 3.0, | |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", | |
| "model": "meltdown", | |
| "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\gsm8k\\gsm8k.yaml" | |
| } | |
| }, | |
| "mmlu_high_school_mathematics_generative": { | |
| "task": "mmlu_high_school_mathematics_generative", | |
| "task_alias": "high_school_mathematics", | |
| "dataset_path": "cais/mmlu", | |
| "dataset_name": "high_school_mathematics", | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}", | |
| "unsafe_code": false, | |
| "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n", | |
| "split": "dev", | |
| "process_docs": null, | |
| "fewshot_indices": null, | |
| "samples": null, | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_choice": null, | |
| "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}", | |
| "gen_prefix": null, | |
| "fewshot_delimiter": "\n\n", | |
| "target_delimiter": " " | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "exact_match", | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "ignore_punctuation": true, | |
| "ignore_case": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [ | |
| "</s>", | |
| "\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "get_response", | |
| "filter": [ | |
| { | |
| "function": "regex", | |
| "regex_pattern": "^(.*?)(?=\\n|$)" | |
| }, | |
| { | |
| "function": "remove_whitespace" | |
| }, | |
| { | |
| "function": "regex", | |
| "regex_pattern": "^(.*?)\\s*$" | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 3.0, | |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", | |
| "model": "meltdown", | |
| "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\mmlu\\generative\\mmlu_high_school_mathematics.yaml" | |
| } | |
| }, | |
| "mmlu_high_school_computer_science_generative": { | |
| "task": "mmlu_high_school_computer_science_generative", | |
| "task_alias": "high_school_computer_science", | |
| "dataset_path": "cais/mmlu", | |
| "dataset_name": "high_school_computer_science", | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}", | |
| "unsafe_code": false, | |
| "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n", | |
| "split": "dev", | |
| "process_docs": null, | |
| "fewshot_indices": null, | |
| "samples": null, | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_choice": null, | |
| "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}", | |
| "gen_prefix": null, | |
| "fewshot_delimiter": "\n\n", | |
| "target_delimiter": " " | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "exact_match", | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "ignore_punctuation": true, | |
| "ignore_case": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [ | |
| "</s>", | |
| "\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "get_response", | |
| "filter": [ | |
| { | |
| "function": "regex", | |
| "regex_pattern": "^(.*?)(?=\\n|$)" | |
| }, | |
| { | |
| "function": "remove_whitespace" | |
| }, | |
| { | |
| "function": "regex", | |
| "regex_pattern": "^(.*?)\\s*$" | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 3.0, | |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", | |
| "model": "meltdown", | |
| "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\mmlu\\generative\\mmlu_high_school_computer_science.yaml" | |
| } | |
| }, | |
| "ifeval": { | |
| "task": "ifeval", | |
| "dataset_path": "google/IFEval", | |
| "test_split": "train", | |
| "doc_to_text": "prompt", | |
| "doc_to_target": 0, | |
| "unsafe_code": false, | |
| "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "default", | |
| "split": null, | |
| "process_docs": null, | |
| "fewshot_indices": null, | |
| "samples": null, | |
| "doc_to_text": "prompt", | |
| "doc_to_choice": null, | |
| "doc_to_target": 0, | |
| "gen_prefix": null, | |
| "fewshot_delimiter": "\n\n", | |
| "target_delimiter": " " | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "prompt_level_strict_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_strict_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "prompt_level_loose_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_loose_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [], | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "max_gen_toks": 1280 | |
| }, | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 4.0, | |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", | |
| "model": "meltdown", | |
| "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\ifeval\\ifeval.yaml" | |
| } | |
| } | |
| } | |
| } |