test/run_eval.py

"""
Usage:
python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10
"""

import argparse
import json
import os
import time

from sglang.test.simple_eval_common import (
    ChatCompletionSampler,
    Eval,
    make_report,
    set_ulimit,
)


def get_thinking_kwargs(args):
    thinking_mode = getattr(args, "thinking_mode", None)
    if thinking_mode in THINKING_MODE_CHOICES:
        if thinking_mode == "deepseek-v3":
            thinking_param = "thinking"
        else:
            thinking_param = "enable_thinking"
        return {
            "chat_template_kwargs": {thinking_param: True},
        }
    return {}


def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
    # Get thinking kwargs based on user's choice
    thinking_kwargs = get_thinking_kwargs(args)

    sampler = ChatCompletionSampler(
        model=args.model,
        max_tokens=getattr(args, "max_tokens", 2048),
        base_url=base_url,
        temperature=getattr(args, "temperature", 0.0),
        reasoning_effort=getattr(args, "reasoning_effort", None),
        extra_body=thinking_kwargs,
    )

    # Run eval
    tic = time.perf_counter()
    result = eval_obj(sampler)
    latency = time.perf_counter() - tic

    return result, latency, sampler


def run_eval(args):
    set_ulimit()

    if "OPENAI_API_KEY" not in os.environ:
        os.environ["OPENAI_API_KEY"] = "EMPTY"

    base_url = (
        f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
    )

    if args.eval_name == "mmlu":
        from sglang.test.simple_eval_mmlu import MMLUEval

        filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
        eval_obj = MMLUEval(filename, args.num_examples, args.num_threads)
    elif args.eval_name == "math":
        from sglang.test.simple_eval_math import MathEval

        equality_checker = ChatCompletionSampler(model="gpt-4-turbo")

        filename = (
            "https://openaipublic.blob.core.windows.net/simple-evals/math_test.csv"
        )
        eval_obj = MathEval(
            filename, equality_checker, args.num_examples, args.num_threads
        )
    elif args.eval_name == "mgsm":
        from sglang.test.simple_eval_mgsm import MGSMEval

        eval_obj = MGSMEval(args.num_examples, args.num_threads)
    elif args.eval_name == "mgsm_en":
        from sglang.test.simple_eval_mgsm import MGSMEval

        eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"])
    elif args.eval_name == "gpqa":
        from sglang.test.simple_eval_gpqa import GPQAEval

        filename = (
            "https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv"
        )
        eval_obj = GPQAEval(filename, args.num_examples, args.num_threads)
    elif args.eval_name == "humaneval":
        from sglang.test.simple_eval_humaneval import HumanEval

        eval_obj = HumanEval(args.num_examples, args.num_threads)
    elif args.eval_name == "longbench_v2":
        from sglang.test.simple_eval_longbench_v2 import LongBenchV2Eval

        # Default to HuggingFace dataset, can be overridden with --dataset-path
        data_source = args.dataset_path
        categories = args.categories.split(",") if args.categories else None

        eval_obj = LongBenchV2Eval(
            model=args.model,
            data_source=data_source,
            num_examples=args.num_examples,
            num_threads=args.num_threads,
            categories=categories,
            max_context_length=getattr(args, "max_context_length", None),
            min_context_length=getattr(args, "min_context_length", None),
        )
    elif args.eval_name == "mmmu":
        # VLM MMMU evaluation with fixed 100 examples by default
        from sglang.test.simple_eval_mmmu_vlm import MMMUVLMEval

        eval_obj = MMMUVLMEval(args.num_examples, args.num_threads)
    else:
        raise ValueError(f"Invalid eval name: {args.eval_name}")

    if getattr(args, "repeat", 1) == 1:
        result, latency, sampler = run_eval_once(args, base_url, eval_obj)
    else:
        from concurrent.futures import ThreadPoolExecutor

        executor = ThreadPoolExecutor(max_workers=args.repeat)

        futures = [
            executor.submit(run_eval_once, args, base_url, eval_obj)
            for _ in range(args.repeat)
        ]

        scores_repeat = []

        for f in futures:
            result, latency, sampler = f.result()
            scores_repeat.append(result.score)

        mean_score = sum(scores_repeat) / len(scores_repeat)
        scores_repeat = [f"{s:.3f}" for s in scores_repeat]
        print("=" * 20)
        print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
        print(f"Scores: {scores_repeat}")
        print("=" * 20)

        executor.shutdown()

    # Dump reports
    metrics = result.metrics | {"score": result.score}
    file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
    report_filename = f"/tmp/{file_stem}.html"
    print(f"Writing report to {report_filename}")
    with open(report_filename, "w") as fh:
        fh.write(make_report(result))
    metrics = result.metrics | {"score": result.score}
    print(metrics)
    result_filename = f"/tmp/{file_stem}.json"
    with open(result_filename, "w") as f:
        f.write(json.dumps(metrics, indent=2))
    print(f"Writing results to {result_filename}")

    # Print results
    print(f"Total latency: {latency:.3f} s")
    print(f"Score: {metrics['score']:.3f}")

    if getattr(args, "return_latency", False):
        return metrics, latency
    return metrics


THINKING_MODE_CHOICES = ["deepseek-r1", "deepseek-v3", "qwen3"]

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--base-url",
        type=str,
        default=None,
        help="Server or API base url if not using http host and port.",
    )
    parser.add_argument(
        "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0."
    )
    parser.add_argument(
        "--port",
        type=int,
        help="If not set, the default port is configured according to its default value for different LLM Inference Engines.",
    )
    parser.add_argument(
        "--model",
        type=str,
        help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
    )
    parser.add_argument(
        "--repeat", type=int, default=1, help="repeat the evaluation n times"
    )
    parser.add_argument("--eval-name", type=str, default="mmlu")
    parser.add_argument("--num-examples", type=int)
    parser.add_argument("--num-threads", type=int, default=512)
    parser.add_argument("--max-tokens", type=int, default=2048)
    parser.add_argument("--temperature", type=float, default=0.0)
    parser.add_argument("--reasoning-effort", type=str)
    parser.add_argument(
        "--thinking-mode",
        default=None,
        type=str,
        choices=THINKING_MODE_CHOICES,
        help="Enable thinking mode in Deepseek R1, V3.1/3.2, or Qwen3",
    )

    # LongBench-v2 specific arguments
    parser.add_argument(
        "--dataset-path",
        type=str,
        default="THUDM/LongBench-v2",
        help="Path to dataset file or HuggingFace dataset name for LongBench-v2",
    )
    parser.add_argument(
        "--categories",
        type=str,
        default=None,
        help="Comma-separated list of categories to evaluate for LongBench-v2",
    )
    parser.add_argument(
        "--max-context-length",
        type=int,
        help="Maximum context length in characters for LongBench-v2",
    )
    parser.add_argument(
        "--min-context-length",
        type=int,
        help="Minimum context length in characters for LongBench-v2",
    )

    args = parser.parse_args()

    run_eval(args)