{
  "schema_version": "0.2.2",
  "evaluation_id": "mmlu_pro_chat/RedHatAI/MiniMax-M2.5-quantized.w4a16/1777302998.11639",
  "evaluation_timestamp": "1777037460",
  "retrieved_timestamp": "1777302998.11639",
  "source_metadata": {
    "source_name": "lm-evaluation-harness",
    "source_type": "evaluation_run",
    "source_organization_name": "RedHatAI",
    "evaluator_relationship": "third_party"
  },
  "eval_library": {
    "name": "lm_eval",
    "version": "0.4.12.dev0"
  },
  "model_info": {
    "name": "RedHatAI/MiniMax-M2.5-quantized.w4a16",
    "id": "RedHatAI/MiniMax-M2.5-quantized.w4a16",
    "developer": "RedHatAI",
    "additional_details": {
      "model_args": "{'model': 'RedHatAI/MiniMax-M2.5-quantized.w4a16', 'max_length': 196608, 'base_url': 'http://0.0.0.0:8000/v1/chat/completions', 'num_concurrent': 28, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 2400}",
      "seed": "1234",
      "num_seeds_merged": "3"
    }
  },
  "evaluation_results": [
    {
      "evaluation_name": "mmlu_pro_chat_biology/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.9079497907949791,
        "details": {
          "seed_scores": "[0.9093444909344491, 0.9065550906555091, 0.9079497907949791]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0008052305009618233,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_business/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8719898605830165,
        "details": {
          "seed_scores": "[0.8694550063371356, 0.8719898605830165, 0.8745247148288974]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0014634987812158046,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_chemistry/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8695524146054181,
        "details": {
          "seed_scores": "[0.8692579505300353, 0.8719081272084805, 0.8674911660777385]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0012835391470967918,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_computer_science/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8601626016260162,
        "details": {
          "seed_scores": "[0.8658536585365854, 0.8609756097560975, 0.8536585365853658]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.003543820279301364,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_economics/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8499210110584517,
        "details": {
          "seed_scores": "[0.8542654028436019, 0.8483412322274881, 0.8471563981042654]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0021989590690481933,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_engineering/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7282421740626075,
        "details": {
          "seed_scores": "[0.7347781217750258, 0.7213622291021672, 0.7285861713106295]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.003876652105120269,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_health/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7921760391198044,
        "details": {
          "seed_scores": "[0.7946210268948656, 0.7970660146699267, 0.784841075794621]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0037347805174864275,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_history/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.6815398075240595,
        "details": {
          "seed_scores": "[0.6850393700787402, 0.6797900262467191, 0.6797900262467191]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0017497812773403416,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_law/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.5812897366030881,
        "details": {
          "seed_scores": "[0.5876475930971844, 0.5767484105358764, 0.5794732061762035]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0032747967987865603,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_math/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.9346163335800641,
        "details": {
          "seed_scores": "[0.9333826794966691, 0.9370836417468542, 0.9333826794966691]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0012336540833950418,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_other/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7853535353535354,
        "details": {
          "seed_scores": "[0.7954545454545454, 0.775974025974026, 0.7846320346320347]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.005635100776267413,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_philosophy/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.7247828991315964,
        "details": {
          "seed_scores": "[0.7234468937875751, 0.7274549098196392, 0.7234468937875751]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0013360053440213775,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_physics/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8734924300744162,
        "details": {
          "seed_scores": "[0.8775981524249422, 0.8729792147806005, 0.8698999230177059]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.002237053602022402,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat_psychology/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "hf_dataset",
        "hf_repo": "TIGER-Lab/MMLU-Pro",
        "hf_split": "test"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8074352548036758,
        "details": {
          "seed_scores": "[0.7944862155388471, 0.8233082706766918, 0.8045112781954887]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.008447681042671974,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      },
      "generation_config": {
        "generation_args": {
          "temperature": 1.0,
          "top_p": 0.95,
          "top_k": 40.0,
          "max_tokens": 64000,
          "max_attempts": 1
        },
        "additional_details": {
          "until": "[]",
          "do_sample": "true",
          "min_p": "0.0",
          "presence_penalty": "1.5",
          "repetition_penalty": "1.0",
          "seed": "1234",
          "num_fewshot": "0"
        }
      }
    },
    {
      "evaluation_name": "mmlu_pro_chat/custom-extract",
      "source_data": {
        "dataset_name": "mmlu_pro_chat",
        "source_type": "other"
      },
      "evaluation_timestamp": "1777088443",
      "metric_config": {
        "evaluation_description": "exact_match (filter: custom-extract)",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.8124722960992908,
        "details": {
          "seed_scores": "[0.8144115691489362, 0.8125, 0.8105053191489362]",
          "seed_values": "[1234, 4158, 42]"
        },
        "uncertainty": {
          "standard_error": {
            "value": 0.0011277223203151384,
            "method": "across_seeds"
          },
          "num_samples": 3
        }
      }
    }
  ]
}