| { |
| "results": { |
| "gsm8k": { |
| "name": "gsm8k", |
| "alias": "gsm8k", |
| "sample_len": 200, |
| "exact_match,strict-match": 0.69, |
| "exact_match_stderr,strict-match": 0.032785276754349606, |
| "exact_match,flexible-extract": 0.965, |
| "exact_match_stderr,flexible-extract": 0.013027801736688036 |
| }, |
| "ifeval": { |
| "name": "ifeval", |
| "alias": "ifeval", |
| "sample_len": 200, |
| "prompt_level_strict_acc,none": 0.735, |
| "prompt_level_strict_acc_stderr,none": 0.03128528159088722, |
| "inst_level_strict_acc,none": 0.8144654088050315, |
| "inst_level_strict_acc_stderr,none": "N/A", |
| "prompt_level_loose_acc,none": 0.765, |
| "prompt_level_loose_acc_stderr,none": 0.030056479497755487, |
| "inst_level_loose_acc,none": 0.839622641509434, |
| "inst_level_loose_acc_stderr,none": "N/A" |
| } |
| }, |
| "group_subtasks": {}, |
| "configs": { |
| "gsm8k": { |
| "task": "gsm8k", |
| "dataset_path": "openai/gsm8k", |
| "dataset_name": "main", |
| "training_split": "train", |
| "test_split": "test", |
| "fewshot_split": "train", |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_target": "{{answer}}", |
| "unsafe_code": false, |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": "train", |
| "process_docs": null, |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "Question: {{question}}\nAnswer:", |
| "doc_to_choice": null, |
| "doc_to_target": "{{answer}}", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 5, |
| "metric_list": [ |
| { |
| "metric": "exact_match", |
| "aggregation": "mean", |
| "higher_is_better": true, |
| "ignore_case": true, |
| "ignore_punctuation": false, |
| "regexes_to_ignore": [ |
| ",", |
| "\\$", |
| "(?s).*#### ", |
| "\\.$" |
| ] |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "Question:", |
| "</s>", |
| "<|im_end|>" |
| ], |
| "do_sample": false, |
| "temperature": 0.0 |
| }, |
| "repeats": 1, |
| "filter_list": [ |
| { |
| "name": "strict-match", |
| "filter": [ |
| { |
| "function": "regex", |
| "regex_pattern": "#### (\\-?[0-9\\.\\,]+)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| }, |
| { |
| "name": "flexible-extract", |
| "filter": [ |
| { |
| "function": "regex", |
| "group_select": -1, |
| "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)" |
| }, |
| { |
| "function": "take_first" |
| } |
| ] |
| } |
| ], |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 3.0, |
| "model": "dsv4-hybrid", |
| "base_url": "http://localhost:PORT/v1/chat/completions", |
| "num_concurrent": 16, |
| "max_retries": 3, |
| "tokenized_requests": false, |
| "tokenizer": "<model_path>", |
| "tokenizer_backend": "huggingface", |
| "config_source": "<lm_eval>/tasks/gsm8k/gsm8k.yaml" |
| } |
| }, |
| "ifeval": { |
| "task": "ifeval", |
| "dataset_path": "google/IFEval", |
| "test_split": "train", |
| "doc_to_text": "prompt", |
| "doc_to_target": 0, |
| "unsafe_code": false, |
| "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": null, |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "prompt", |
| "doc_to_choice": null, |
| "doc_to_target": 0, |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "prompt_level_strict_acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "inst_level_strict_acc", |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "prompt_level_loose_acc", |
| "aggregation": "mean", |
| "higher_is_better": true |
| }, |
| { |
| "metric": "inst_level_loose_acc", |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", |
| "higher_is_better": true |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [], |
| "do_sample": false, |
| "temperature": 0.0, |
| "max_gen_toks": 1280 |
| }, |
| "repeats": 1, |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 4.0, |
| "model": "dsv4-hybrid", |
| "base_url": "http://localhost:PORT/v1/chat/completions", |
| "num_concurrent": 16, |
| "max_retries": 3, |
| "tokenized_requests": false, |
| "tokenizer": "<model_path>", |
| "tokenizer_backend": "huggingface", |
| "config_source": "<lm_eval>/tasks/ifeval/ifeval.yaml" |
| } |
| } |
| }, |
| "versions": { |
| "gsm8k": 3.0, |
| "ifeval": 4.0 |
| }, |
| "n-shot": { |
| "gsm8k": 5, |
| "ifeval": 0 |
| }, |
| "higher_is_better": { |
| "gsm8k": { |
| "exact_match": true |
| }, |
| "ifeval": { |
| "prompt_level_strict_acc": true, |
| "inst_level_strict_acc": true, |
| "prompt_level_loose_acc": true, |
| "inst_level_loose_acc": true |
| } |
| }, |
| "n-samples": { |
| "gsm8k": { |
| "original": 1319, |
| "effective": 200 |
| }, |
| "ifeval": { |
| "original": 541, |
| "effective": 200 |
| } |
| }, |
| "config": { |
| "model": "local-chat-completions", |
| "model_args": { |
| "model": "dsv4-hybrid", |
| "base_url": "http://localhost:PORT/v1/chat/completions", |
| "num_concurrent": 16, |
| "max_retries": 3, |
| "tokenized_requests": false, |
| "tokenizer": "<model_path>", |
| "tokenizer_backend": "huggingface" |
| }, |
| "batch_size": 1, |
| "batch_sizes": [], |
| "device": "cuda:0", |
| "use_cache": null, |
| "limit": 200.0, |
| "bootstrap_iters": 100000, |
| "gen_kwargs": {}, |
| "random_seed": 0, |
| "numpy_seed": 1234, |
| "torch_seed": 1234, |
| "fewshot_seed": 1234 |
| }, |
| "git_hash": null, |
| "date": 1786209999.0285816, |
| "transformers_version": "5.14.1", |
| "lm_eval_version": "0.4.12", |
| "upper_git_hash": null, |
| "tokenizer_pad_token": [ |
| "<｜end▁of▁sentence｜>", |
| "1" |
| ], |
| "tokenizer_eos_token": [ |
| "<｜end▁of▁sentence｜>", |
| "1" |
| ], |
| "tokenizer_bos_token": [ |
| "<｜begin▁of▁sentence｜>", |
| "0" |
| ], |
| "eot_token_id": 1, |
| "max_length": 2047, |
| "task_hashes": {}, |
| "model_source": "local-chat-completions", |
| "model_name": "dsv4-hybrid", |
| "model_name_sanitized": "dsv4-hybrid", |
| "system_instruction": null, |
| "system_instruction_sha": null, |
| "fewshot_as_multiturn": true, |
| "chat_template": null, |
| "chat_template_sha": null, |
| "total_evaluation_time_seconds": "261.7299920649966", |
| "hardware_note": "2x NVIDIA RTX PRO 6000 Blackwell (96GB, SM120), TP=2. Detailed host environment omitted." |
| } |
