{ "results": { "gsm8k": { "name": "gsm8k", "alias": "gsm8k", "sample_len": 200, "exact_match,strict-match": 0.69, "exact_match_stderr,strict-match": 0.032785276754349606, "exact_match,flexible-extract": 0.965, "exact_match_stderr,flexible-extract": 0.013027801736688036 }, "ifeval": { "name": "ifeval", "alias": "ifeval", "sample_len": 200, "prompt_level_strict_acc,none": 0.735, "prompt_level_strict_acc_stderr,none": 0.03128528159088722, "inst_level_strict_acc,none": 0.8144654088050315, "inst_level_strict_acc_stderr,none": "N/A", "prompt_level_loose_acc,none": 0.765, "prompt_level_loose_acc_stderr,none": 0.030056479497755487, "inst_level_loose_acc,none": 0.839622641509434, "inst_level_loose_acc_stderr,none": "N/A" } }, "group_subtasks": {}, "configs": { "gsm8k": { "task": "gsm8k", "dataset_path": "openai/gsm8k", "dataset_name": "main", "training_split": "train", "test_split": "test", "fewshot_split": "train", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{answer}}", "unsafe_code": false, "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": "train", "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": null, "doc_to_target": "{{answer}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 5, "metric_list": [ { "metric": "exact_match", "aggregation": "mean", "higher_is_better": true, "ignore_case": true, "ignore_punctuation": false, "regexes_to_ignore": [ ",", "\\$", "(?s).*#### ", "\\.$" ] } ], "output_type": "generate_until", "generation_kwargs": { "until": [ "Question:", "", "<|im_end|>" ], "do_sample": false, "temperature": 0.0 }, "repeats": 1, "filter_list": [ { "name": "strict-match", "filter": [ { "function": "regex", "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" }, { "function": "take_first" } ] }, { "name": "flexible-extract", "filter": [ { "function": "regex", "group_select": -1, "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" }, { "function": "take_first" } ] } ], "should_decontaminate": false, "metadata": { "version": 3.0, "model": "dsv4-hybrid", "base_url": "http://localhost:PORT/v1/chat/completions", "num_concurrent": 16, "max_retries": 3, "tokenized_requests": false, "tokenizer": "", "tokenizer_backend": "huggingface", "config_source": "/tasks/gsm8k/gsm8k.yaml" } }, "ifeval": { "task": "ifeval", "dataset_path": "google/IFEval", "test_split": "train", "doc_to_text": "prompt", "doc_to_target": 0, "unsafe_code": false, "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "prompt", "doc_to_choice": null, "doc_to_target": 0, "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "prompt_level_strict_acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "inst_level_strict_acc", "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", "higher_is_better": true }, { "metric": "prompt_level_loose_acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "inst_level_loose_acc", "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", "higher_is_better": true } ], "output_type": "generate_until", "generation_kwargs": { "until": [], "do_sample": false, "temperature": 0.0, "max_gen_toks": 1280 }, "repeats": 1, "should_decontaminate": false, "metadata": { "version": 4.0, "model": "dsv4-hybrid", "base_url": "http://localhost:PORT/v1/chat/completions", "num_concurrent": 16, "max_retries": 3, "tokenized_requests": false, "tokenizer": "", "tokenizer_backend": "huggingface", "config_source": "/tasks/ifeval/ifeval.yaml" } } }, "versions": { "gsm8k": 3.0, "ifeval": 4.0 }, "n-shot": { "gsm8k": 5, "ifeval": 0 }, "higher_is_better": { "gsm8k": { "exact_match": true }, "ifeval": { "prompt_level_strict_acc": true, "inst_level_strict_acc": true, "prompt_level_loose_acc": true, "inst_level_loose_acc": true } }, "n-samples": { "gsm8k": { "original": 1319, "effective": 200 }, "ifeval": { "original": 541, "effective": 200 } }, "config": { "model": "local-chat-completions", "model_args": { "model": "dsv4-hybrid", "base_url": "http://localhost:PORT/v1/chat/completions", "num_concurrent": 16, "max_retries": 3, "tokenized_requests": false, "tokenizer": "", "tokenizer_backend": "huggingface" }, "batch_size": 1, "batch_sizes": [], "device": "cuda:0", "use_cache": null, "limit": 200.0, "bootstrap_iters": 100000, "gen_kwargs": {}, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": null, "date": 1786209999.0285816, "transformers_version": "5.14.1", "lm_eval_version": "0.4.12", "upper_git_hash": null, "tokenizer_pad_token": [ "<｜end▁of▁sentence｜>", "1" ], "tokenizer_eos_token": [ "<｜end▁of▁sentence｜>", "1" ], "tokenizer_bos_token": [ "<｜begin▁of▁sentence｜>", "0" ], "eot_token_id": 1, "max_length": 2047, "task_hashes": {}, "model_source": "local-chat-completions", "model_name": "dsv4-hybrid", "model_name_sanitized": "dsv4-hybrid", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": true, "chat_template": null, "chat_template_sha": null, "total_evaluation_time_seconds": "261.7299920649966", "hardware_note": "2x NVIDIA RTX PRO 6000 Blackwell (96GB, SM120), TP=2. Detailed host environment omitted." }