{
  "schema_version": "0.2.2",
  "evaluation_id": "mmlu_pro_chat/RedHatAI/MiniMax-M2.5-NVFP4/1777381372.569359",
  "evaluation_timestamp": "1776209745",
  "retrieved_timestamp": "1777381372.569359",
  "source_metadata": {
    "source_name": "lm-evaluation-harness",
    "source_type": "evaluation_run",
    "source_organization_name": "RedHatAI",
    "evaluator_relationship": "third_party"
  },
  "eval_library": {
    "name": "lm_eval",
    "version": "0.4.12.dev0"
  },
  "model_info": {
    "name": "RedHatAI/MiniMax-M2.5-NVFP4",
    "id": "RedHatAI/MiniMax-M2.5-NVFP4",
    "developer": "RedHatAI",
    "additional_details": {
      "model_args": "{'model': 'RedHatAI/MiniMax-M2.5-NVFP4', 'max_length': 196608, 'base_url': 'http://0.0.0.0:8000/v1/chat/completions', 'num_concurrent': 128, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 2400}",
      "seed": "1234",
      "num_seeds_merged": "3"
    }
  },
  "evaluation_results": [
    {
      "evaluation_name": "mmlu_pro_chat_biology/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.903300790330079,
        "details": {
          "seed_scores": "[0.9121338912133892, 0.9037656903765691, 0.8940027894002789]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.005239157447505677,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_business/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8588931136459653,
        "details": {
          "seed_scores": "[0.8605830164765526, 0.8504435994930292, 0.8656527249683144]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.004471062629597958,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_chemistry/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8674911660777385,
        "details": {
          "seed_scores": "[0.8666077738515902, 0.8683745583038869, 0.8674911660777385]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0005100267395667788,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_computer_science/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8569105691056911,
        "details": {
          "seed_scores": "[0.875609756097561, 0.8512195121951219, 0.8439024390243902]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.009585224489879356,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_economics/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8404423380726699,
        "details": {
          "seed_scores": "[0.8376777251184834, 0.840047393364929, 0.8436018957345972]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.001721524069328884,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_engineering/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7220502235982112,
        "details": {
          "seed_scores": "[0.7409700722394221, 0.6996904024767802, 0.7254901960784313]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.012039903680770573,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_health/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7766911165444174,
        "details": {
          "seed_scores": "[0.7701711491442543, 0.78239608801956, 0.7775061124694377]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0035524848765612657,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_history/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.6500437445319335,
        "details": {
          "seed_scores": "[0.6272965879265092, 0.6509186351706037, 0.6719160104986877]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.01288794388683834,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_law/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.55858310626703,
        "details": {
          "seed_scores": "[0.5640326975476839, 0.5404178019981835, 0.5712988192552225]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.00932171015340034,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_math/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.9338761411300271,
        "details": {
          "seed_scores": "[0.9341228719467062, 0.9356032568467801, 0.9319022945965951]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0010754746961610265,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_other/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7525252525252526,
        "details": {
          "seed_scores": "[0.762987012987013, 0.7435064935064936, 0.7510822510822511]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.005669636957251685,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_philosophy/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.6893787575150301,
        "details": {
          "seed_scores": "[0.6833667334669339, 0.685370741482966, 0.6993987975951904]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0050433095759991695,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_physics/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8763151141904029,
        "details": {
          "seed_scores": "[0.8752886836027713, 0.876058506543495, 0.8775981524249422]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0006789200182357032,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_psychology/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.789891395154553,
        "details": {
          "seed_scores": "[0.7869674185463659, 0.7907268170426065, 0.7919799498746867]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0015060782270108586,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "other"
      },
      "evaluation_timestamp": "1776226127",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8008089539007092,
        "details": {
          "seed_scores": "[0.8029421542553191, 0.7964594414893617, 0.8030252659574468]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.002174888545121904,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      }
    }
  ]
}