{
  "results": {
    "gsm8k": {
      "name": "gsm8k",
      "alias": "gsm8k",
      "sample_len": 20,
      "exact_match,strict-match": 0.65,
      "exact_match_stderr,strict-match": 0.1094243309804831,
      "exact_match,flexible-extract": 0.65,
      "exact_match_stderr,flexible-extract": 0.1094243309804831
    },
    "mmlu_high_school_mathematics_generative": {
      "name": "mmlu_high_school_mathematics_generative",
      "alias": "high_school_mathematics",
      "sample_len": 20,
      "exact_match,get_response": 0.0,
      "exact_match_stderr,get_response": 0.0
    },
    "mmlu_high_school_computer_science_generative": {
      "name": "mmlu_high_school_computer_science_generative",
      "alias": "high_school_computer_science",
      "sample_len": 20,
      "exact_match,get_response": 0.0,
      "exact_match_stderr,get_response": 0.0
    },
    "ifeval": {
      "name": "ifeval",
      "alias": "ifeval",
      "sample_len": 20,
      "prompt_level_strict_acc,none": 0.5,
      "prompt_level_strict_acc_stderr,none": 0.11470786693528086,
      "inst_level_strict_acc,none": 0.6,
      "inst_level_strict_acc_stderr,none": "N/A",
      "prompt_level_loose_acc,none": 0.5,
      "prompt_level_loose_acc_stderr,none": 0.11470786693528086,
      "inst_level_loose_acc,none": 0.6,
      "inst_level_loose_acc_stderr,none": "N/A"
    }
  },
  "configs": {
    "gsm8k": {
      "task": "gsm8k",
      "dataset_path": "openai/gsm8k",
      "dataset_name": "main",
      "training_split": "train",
      "test_split": "test",
      "fewshot_split": "train",
      "doc_to_text": "Question: {{question}}\nAnswer:",
      "doc_to_target": "{{answer}}",
      "unsafe_code": false,
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "fewshot_config": {
        "sampler": "default",
        "split": "train",
        "process_docs": null,
        "fewshot_indices": null,
        "samples": null,
        "doc_to_text": "Question: {{question}}\nAnswer:",
        "doc_to_choice": null,
        "doc_to_target": "{{answer}}",
        "gen_prefix": null,
        "fewshot_delimiter": "\n\n",
        "target_delimiter": " "
      },
      "num_fewshot": 5,
      "metric_list": [
        {
          "metric": "exact_match",
          "aggregation": "mean",
          "higher_is_better": true,
          "ignore_case": true,
          "ignore_punctuation": false,
          "regexes_to_ignore": [
            ",",
            "\\$",
            "(?s).*#### ",
            "\\.$"
          ]
        }
      ],
      "output_type": "generate_until",
      "generation_kwargs": {
        "until": [
          "Question:",
          "</s>",
          "<|im_end|>"
        ],
        "do_sample": false,
        "temperature": 0.0
      },
      "repeats": 1,
      "filter_list": [
        {
          "name": "strict-match",
          "filter": [
            {
              "function": "regex",
              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
            },
            {
              "function": "take_first"
            }
          ]
        },
        {
          "name": "flexible-extract",
          "filter": [
            {
              "function": "regex",
              "group_select": -1,
              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
            },
            {
              "function": "take_first"
            }
          ]
        }
      ],
      "should_decontaminate": false,
      "metadata": {
        "version": 3.0,
        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
        "model": "meltdown",
        "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\gsm8k\\gsm8k.yaml"
      }
    },
    "mmlu_high_school_mathematics_generative": {
      "task": "mmlu_high_school_mathematics_generative",
      "task_alias": "high_school_mathematics",
      "dataset_path": "cais/mmlu",
      "dataset_name": "high_school_mathematics",
      "test_split": "test",
      "fewshot_split": "dev",
      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
      "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}",
      "unsafe_code": false,
      "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "fewshot_config": {
        "sampler": "first_n",
        "split": "dev",
        "process_docs": null,
        "fewshot_indices": null,
        "samples": null,
        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
        "doc_to_choice": null,
        "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}",
        "gen_prefix": null,
        "fewshot_delimiter": "\n\n",
        "target_delimiter": " "
      },
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "exact_match",
          "aggregation": "mean",
          "higher_is_better": true,
          "ignore_punctuation": true,
          "ignore_case": true
        }
      ],
      "output_type": "generate_until",
      "generation_kwargs": {
        "until": [
          "</s>",
          "\n"
        ]
      },
      "repeats": 1,
      "filter_list": [
        {
          "name": "get_response",
          "filter": [
            {
              "function": "regex",
              "regex_pattern": "^(.*?)(?=\\n|$)"
            },
            {
              "function": "remove_whitespace"
            },
            {
              "function": "regex",
              "regex_pattern": "^(.*?)\\s*$"
            },
            {
              "function": "take_first"
            }
          ]
        }
      ],
      "should_decontaminate": false,
      "metadata": {
        "version": 3.0,
        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
        "model": "meltdown",
        "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\mmlu\\generative\\mmlu_high_school_mathematics.yaml"
      }
    },
    "mmlu_high_school_computer_science_generative": {
      "task": "mmlu_high_school_computer_science_generative",
      "task_alias": "high_school_computer_science",
      "dataset_path": "cais/mmlu",
      "dataset_name": "high_school_computer_science",
      "test_split": "test",
      "fewshot_split": "dev",
      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
      "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}",
      "unsafe_code": false,
      "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "fewshot_config": {
        "sampler": "first_n",
        "split": "dev",
        "process_docs": null,
        "fewshot_indices": null,
        "samples": null,
        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
        "doc_to_choice": null,
        "doc_to_target": "{{['A', 'B', 'C', 'D'][answer]}}",
        "gen_prefix": null,
        "fewshot_delimiter": "\n\n",
        "target_delimiter": " "
      },
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "exact_match",
          "aggregation": "mean",
          "higher_is_better": true,
          "ignore_punctuation": true,
          "ignore_case": true
        }
      ],
      "output_type": "generate_until",
      "generation_kwargs": {
        "until": [
          "</s>",
          "\n"
        ]
      },
      "repeats": 1,
      "filter_list": [
        {
          "name": "get_response",
          "filter": [
            {
              "function": "regex",
              "regex_pattern": "^(.*?)(?=\\n|$)"
            },
            {
              "function": "remove_whitespace"
            },
            {
              "function": "regex",
              "regex_pattern": "^(.*?)\\s*$"
            },
            {
              "function": "take_first"
            }
          ]
        }
      ],
      "should_decontaminate": false,
      "metadata": {
        "version": 3.0,
        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
        "model": "meltdown",
        "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\mmlu\\generative\\mmlu_high_school_computer_science.yaml"
      }
    },
    "ifeval": {
      "task": "ifeval",
      "dataset_path": "google/IFEval",
      "test_split": "train",
      "doc_to_text": "prompt",
      "doc_to_target": 0,
      "unsafe_code": false,
      "process_results": "def process_results(doc, results):\n    inp = InputExample(\n        key=doc[\"key\"],\n        instruction_id_list=doc[\"instruction_id_list\"],\n        prompt=doc[\"prompt\"],\n        kwargs=doc[\"kwargs\"],\n    )\n    response = results[0]\n\n    out_strict = test_instruction_following_strict(inp, response)\n    out_loose = test_instruction_following_loose(inp, response)\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "fewshot_config": {
        "sampler": "default",
        "split": null,
        "process_docs": null,
        "fewshot_indices": null,
        "samples": null,
        "doc_to_text": "prompt",
        "doc_to_choice": null,
        "doc_to_target": 0,
        "gen_prefix": null,
        "fewshot_delimiter": "\n\n",
        "target_delimiter": " "
      },
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "prompt_level_strict_acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "inst_level_strict_acc",
          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
          "higher_is_better": true
        },
        {
          "metric": "prompt_level_loose_acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "inst_level_loose_acc",
          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
          "higher_is_better": true
        }
      ],
      "output_type": "generate_until",
      "generation_kwargs": {
        "until": [],
        "do_sample": false,
        "temperature": 0.0,
        "max_gen_toks": 1280
      },
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 4.0,
        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
        "model": "meltdown",
        "config_source": "C:\\Users\\jassi\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\ifeval\\ifeval.yaml"
      }
    }
  }
}