| { |
| "gsm8k": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "gsm8k", |
| "alias": "gsm8k", |
| "sample_len": 1319, |
| "exact_match,strict-match": 0.7384382107657316, |
| "exact_match_stderr,strict-match": 0.012105605733382459, |
| "exact_match,flexible-extract": 0.7422289613343442, |
| "exact_match_stderr,flexible-extract": 0.01204837021357661 |
| }, |
| "configs": { |
| "task": "gsm8k", |
| "dataset_path": "openai/gsm8k", |
| "dataset_name": "main", |
| "training_split": "train", |
| "test_split": "test", |
| "fewshot_split": "train", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{answer}}", |
| "unsafe_code": false, |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": "train", |
| "process_docs": null, |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_choice": null, |
| "doc_to_target": "{{answer}}", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "exact_match", |
| "aggregation": "mean", |
| "higher_is_better": true, |
| "ignore_case": true, |
| "ignore_punctuation": false, |
| "regexes_to_ignore": [ |
| ",", |
| "\\$", |
| "(?s).*#### ", |
| "\\.$" |
| ] |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "Question:", |
| "</s>", |
| "<|im_end|>" |
| ], |
| "do_sample": false, |
| "temperature": 0.0 |
| }, |
| "repeats": 1, |
| "filter_list": [ |
| { |
| "name": "strict-match", |
| "filter": [ |
| { |
| "function": "regex", |
| "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| }, |
| { |
| "name": "flexible-extract", |
| "filter": [ |
| { |
| "function": "regex", |
| "group_select": -1, |
| "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| } |
| ], |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 3.0, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml" |
| } |
| } |
| }, |
| "minerva_math500": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "minerva_math500", |
| "alias": "minerva_math500", |
| "sample_len": 500, |
| "exact_match,none": 0.394, |
| "exact_match_stderr,none": 0.021874299301689257, |
| "math_verify,none": 0.434, |
| "math_verify_stderr,none": 0.022187215803029008 |
| }, |
| "configs": { |
| "task": "minerva_math500", |
| "dataset_path": "HuggingFaceH4/MATH-500", |
| "dataset_name": "default", |
| "test_split": "test", |
| "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": normalize_final_answer(\n remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n ),\n }\n if getattr(doc, \"few_shot\", None) is not None:\n out_doc[\"few_shot\"] = True\n return out_doc\n\n return dataset.map(_process_doc)\n", |
| "doc_to_text": "def doc_to_text(doc: dict) -> str:\n return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n", |
| "doc_to_target": "{{answer if few_shot is undefined else solution}}", |
| "unsafe_code": false, |
| "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n candidates = results[0]\n\n unnormalized_answer = get_unnormalized_answer(candidates)\n answer = normalize_final_answer(unnormalized_answer)\n\n if is_equiv(answer, doc[\"answer\"]):\n retval = 1\n else:\n retval = 0\n\n # math_verify\n _mvres = verify(\n gold=parse(doc[\"solution\"]),\n target=parse(candidates),\n )\n mathval = 1 if _mvres else 0\n\n res = {\n \"exact_match\": retval,\n \"math_verify\": mathval,\n }\n return res\n", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "first_n", |
| "split": null, |
| "process_docs": "<function process_docs at 0x7fb63b535760>", |
| "fewshot_indices": null, |
| "samples": "<function list_fewshot_samples at 0x7fb639b6a660>", |
| "doc_to_text": "<function doc_to_text at 0x7fb63a02e700>", |
| "doc_to_choice": null, |
| "doc_to_target": "{{answer if few_shot is undefined else solution}}", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 4, |
| "metric_list": [ |
| { |
| "metric": "exact_match", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "math_verify", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "Problem:" |
| ], |
| "do_sample": false, |
| "temperature": 0.0 |
| }, |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 3.0, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/minerva_math/minerva_math500.yaml" |
| } |
| } |
| }, |
| "bbh_cot_fewshot": { |
| "status": "SUCCESS", |
| "results": { |
| "alias": "bbh_cot_fewshot", |
| "name": "bbh_cot_fewshot", |
| "sample_len": 6511, |
| "exact_match,get-answer": 0.6848410382429734, |
| "exact_match_stderr,get-answer": 0.005162813446797612, |
| "sample_count": { |
| "exact_match,get-answer": 6511 |
| } |
| }, |
| "configs": {} |
| }, |
| "mmlu_pro": { |
| "status": "SUCCESS", |
| "results": { |
| "alias": "mmlu_pro", |
| "name": "mmlu_pro", |
| "sample_len": 12032, |
| "exact_match,custom-extract": 0.4493849734042553, |
| "exact_match_stderr,custom-extract": 0.004369131624528862, |
| "sample_count": { |
| "exact_match,custom-extract": 12032 |
| } |
| }, |
| "configs": {} |
| }, |
| "ifeval": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "ifeval", |
| "alias": "ifeval", |
| "sample_len": 541, |
| "prompt_level_strict_acc,none": 0.38817005545286504, |
| "prompt_level_strict_acc_stderr,none": 0.020971500215794775, |
| "inst_level_strict_acc,none": 0.5071942446043165, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.43622920517560076, |
| "prompt_level_loose_acc_stderr,none": 0.02134085308994027, |
| "inst_level_loose_acc,none": 0.539568345323741, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| }, |
| "configs": { |
| "task": "ifeval", |
| "dataset_path": "google/IFEval", |
| "test_split": "train", |
| "doc_to_text": "prompt", |
| "doc_to_target": 0, |
| "unsafe_code": false, |
| "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": null, |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "prompt", |
| "doc_to_choice": null, |
| "doc_to_target": 0, |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "prompt_level_strict_acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "inst_level_strict_acc", |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "prompt_level_loose_acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "inst_level_loose_acc", |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [], |
| "do_sample": false, |
| "temperature": 0.0, |
| "max_gen_toks": 1280 |
| }, |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 4.0, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/ifeval/ifeval.yaml" |
| } |
| } |
| }, |
| "drop": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "drop", |
| "alias": "drop", |
| "sample_len": 9536, |
| "em,none": 0.0, |
| "em_stderr,none": 0.0, |
| "f1,none": 0.04738255033557058, |
| "f1_stderr,none": 0.001040437534856409 |
| }, |
| "configs": { |
| "task": "drop", |
| "dataset_path": "EleutherAI/drop", |
| "training_split": "train", |
| "validation_split": "validation", |
| "process_docs": "def process_docs(dataset):\n def _process(doc):\n return {\n \"id\": doc[\"query_id\"],\n \"passage\": doc[\"passage\"],\n \"question\": doc[\"question\"],\n \"answers\": get_answers(doc),\n }\n\n return dataset.map(_process)\n", |
| "doc_to_text": "{{passage}} {{question}}", |
| "doc_to_target": "{{ answer|join(',')}}", |
| "unsafe_code": false, |
| "process_results": "def process_results(doc, results):\n preds, golds = results, doc[\"answers\"]\n max_em = 0\n max_f1 = 0\n for gold_answer in golds:\n exact_match, f1_score = get_metrics(preds, gold_answer)\n if gold_answer[0].strip():\n max_em = max(max_em, exact_match)\n max_f1 = max(max_f1, f1_score)\n return {\"em\": max_em, \"f1\": max_f1}\n", |
| "description": "", |
| "target_delimiter": "", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": "<function process_docs at 0x7fd1fa935580>", |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "{{passage}} {{question}}", |
| "doc_to_choice": null, |
| "doc_to_target": "{{ answer|join(',')}}", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": "" |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "em", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "f1", |
| "aggregation": "mean", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "." |
| ] |
| }, |
| "repeats": 1, |
| "should_decontaminate": true, |
| "doc_to_decontamination_query": "{{passage}} {{question}}", |
| "metadata": { |
| "version": 3.0, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/drop/default.yaml" |
| } |
| } |
| }, |
| "hle": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.3284 |
| } |
| }, |
| "ru_mmlu": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.2518 |
| } |
| }, |
| "ru_humaneval": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.23170000000000002 |
| } |
| }, |
| "deep_swe": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.012 |
| } |
| }, |
| "gpqa_diamond_cot_zeroshot": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "gpqa_diamond_cot_zeroshot", |
| "alias": "gpqa_diamond_cot_zeroshot", |
| "sample_len": 198, |
| "exact_match,strict-match": 0.0, |
| "exact_match_stderr,strict-match": 0.0, |
| "exact_match,flexible-extract": 0.13131313131313133, |
| "exact_match_stderr,flexible-extract": 0.024063156416822527 |
| }, |
| "configs": { |
| "task": "gpqa_diamond_cot_zeroshot", |
| "dataset_path": "Idavidrein/gpqa", |
| "dataset_name": "gpqa_diamond", |
| "training_split": "train", |
| "validation_split": "train", |
| "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n random.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"choices\": [choices[0], choices[1], choices[2], choices[3]],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", |
| "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": "<function process_docs at 0x7efb8f515620>", |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ", |
| "doc_to_choice": null, |
| "doc_to_target": "answer", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "exact_match", |
| "aggregation": "mean", |
| "higher_is_better": true, |
| "ignore_case": true, |
| "ignore_punctuation": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "</s>" |
| ], |
| "do_sample": false, |
| "temperature": 0.0 |
| }, |
| "repeats": 1, |
| "filter_list": [ |
| { |
| "name": "strict-match", |
| "filter": [ |
| { |
| "function": "regex", |
| "regex_pattern": "(?<=The answer is )(.*)(?=.)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| }, |
| { |
| "name": "flexible-extract", |
| "filter": [ |
| { |
| "function": "multi_choice_regex", |
| "group_select": -1, |
| "ignore_case": true, |
| "ignore_punctuation": true, |
| "regex_pattern": "(\\([A-Z]\\))" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| } |
| ], |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 2.2, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_diamond_cot_zeroshot.yaml" |
| } |
| } |
| }, |
| "gpqa_main_cot_zeroshot": { |
| "status": "SUCCESS", |
| "results": { |
| "name": "gpqa_main_cot_zeroshot", |
| "alias": "gpqa_main_cot_zeroshot", |
| "sample_len": 448, |
| "exact_match,strict-match": 0.004464285714285714, |
| "exact_match_stderr,strict-match": 0.0031531937104656623, |
| "exact_match,flexible-extract": 0.19642857142857142, |
| "exact_match_stderr,flexible-extract": 0.018791472419524242 |
| }, |
| "configs": { |
| "task": "gpqa_main_cot_zeroshot", |
| "dataset_path": "Idavidrein/gpqa", |
| "dataset_name": "gpqa_main", |
| "training_split": "train", |
| "validation_split": "train", |
| "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n choices = [\n preprocess(doc[\"Incorrect Answer 1\"]),\n preprocess(doc[\"Incorrect Answer 2\"]),\n preprocess(doc[\"Incorrect Answer 3\"]),\n preprocess(doc[\"Correct Answer\"]),\n ]\n\n random.shuffle(choices)\n correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n out_doc = {\n \"choice1\": choices[0],\n \"choice2\": choices[1],\n \"choice3\": choices[2],\n \"choice4\": choices[3],\n \"choices\": [choices[0], choices[1], choices[2], choices[3]],\n \"answer\": f\"({chr(65 + correct_answer_index)})\",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", |
| "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ", |
| "doc_to_target": "answer", |
| "unsafe_code": false, |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": "<function process_docs at 0x7fa5fb545620>", |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ", |
| "doc_to_choice": null, |
| "doc_to_target": "answer", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "exact_match", |
| "aggregation": "mean", |
| "higher_is_better": true, |
| "ignore_case": true, |
| "ignore_punctuation": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "</s>" |
| ], |
| "do_sample": false, |
| "temperature": 0.0 |
| }, |
| "repeats": 1, |
| "filter_list": [ |
| { |
| "name": "strict-match", |
| "filter": [ |
| { |
| "function": "regex", |
| "regex_pattern": "(?<=The answer is )(.*)(?=.)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| }, |
| { |
| "name": "flexible-extract", |
| "filter": [ |
| { |
| "function": "multi_choice_regex", |
| "group_select": -1, |
| "ignore_case": true, |
| "ignore_punctuation": true, |
| "regex_pattern": "(\\([A-Z]\\))" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| } |
| ], |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 2.2, |
| "model": "quartz_r1_genesis_clean", |
| "base_url": "http://127.0.0.1:8080/v1/chat/completions", |
| "num_concurrent": 2, |
| "max_retries": 3, |
| "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_main_cot_zeroshot.yaml" |
| } |
| } |
| }, |
| "arc_challenge": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.8676999999999999 |
| } |
| }, |
| "hellaswag": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.7190000000000001 |
| } |
| }, |
| "winogrande": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.49090000000000006 |
| } |
| }, |
| "truthfulqa_mc2": { |
| "status": "SUCCESS", |
| "results": { |
| "acc,none": 0.2827 |
| } |
| } |
| } |
