benchmark/evalplus_benchmark.py

#!/usr/bin/env python3
"""EvalPlus HumanEval+ benchmark for adaptive-operator-v4.1.

Runs only our model. Comparison scores come from the public EvalPlus leaderboard:
  https://evalplus.github.io/leaderboard.html

This makes results immediately comparable without re-running other models.
"""

import json
import os
import re
import subprocess
import sys
import time
from pathlib import Path

RESULTS_DIR = Path("/root/training/evalplus_results")
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
CHART_PATH = RESULTS_DIR / "benchmark_comparison.png"
JSON_PATH = RESULTS_DIR / "benchmark_results.json"

# Our model
OUR_MODEL = "/dev/shm/merged_model"

# Published EvalPlus HumanEval+ pass@1 scores (greedy/temp=0)
# Source: https://evalplus.github.io/leaderboard.html (as of 2025)
PUBLISHED_SCORES = {
    "Qwen2.5-Coder-7B-Instruct": 68.9,
    "Qwen2.5-Coder-3B-Instruct": 62.2,
    "DeepSeek-Coder-6.7B-Instruct": 71.6,
    "Qwen2.5-7B-Instruct": 49.4,
    "Qwen3-8B": 65.2,
    "Llama-3.1-8B-Instruct": 47.6,
    "GPT-4o": 80.5,
    "Claude-3.5-Sonnet": 81.7,
}


def run_evalplus(model_path: str) -> dict:
    """Run EvalPlus HumanEval+ on our model."""
    cmd = [
        "python3", "-m", "evalplus.evaluate",
        "--model", model_path,
        "--dataset", "humaneval",
        "--backend", "vllm",
        "--greedy",
    ]
    
    print(f"\n{'='*60}")
    print(f"Running EvalPlus HumanEval+ on: {model_path}")
    print(f"Command: {' '.join(cmd)}")
    print(f"{'='*60}\n", flush=True)
    
    t0 = time.time()
    result = subprocess.run(
        cmd,
        capture_output=True,
        text=True,
        timeout=3600,
        env={**os.environ, "HF_TOKEN": os.environ.get("HF_TOKEN", "")},
    )
    elapsed = time.time() - t0
    
    # Parse pass@1 from output
    pass_at_1 = None
    # EvalPlus prints something like "humaneval plus  pass@1: 68.9"
    for line in result.stdout.split("\n"):
        if "pass@1" in line.lower():
            match = re.search(r"pass@1[:\s]+([\d.]+)", line, re.IGNORECASE)
            if match:
                pass_at_1 = float(match.group(1))
                break
    
    # Also check for "plus" and "base" separately
    plus_score = None
    base_score = None
    for line in result.stdout.split("\n"):
        if "plus" in line.lower() and "pass@1" in line.lower():
            match = re.search(r"([\d.]+)", line.split("pass@1")[-1])
            if match:
                plus_score = float(match.group(1))
        if "base" in line.lower() and "pass@1" in line.lower():
            match = re.search(r"([\d.]+)", line.split("pass@1")[-1])
            if match:
                base_score = float(match.group(1))
    
    return {
        "model_path": model_path,
        "pass_at_1": pass_at_1,
        "plus_pass_at_1": plus_score,
        "base_pass_at_1": base_score,
        "elapsed_s": elapsed,
        "stdout": result.stdout,
        "stderr": result.stderr[-1000:] if result.stderr else "",
        "returncode": result.returncode,
    }


def generate_chart(our_score: float) -> None:
    """Generate comparison chart with published scores."""
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    import numpy as np
    
    # Combine our score with published scores
    all_models = {
        "Adaptive Operator v4.1 (ours)": our_score,
    }
    all_models.update(PUBLISHED_SCORES)
    
    # Sort by score descending
    sorted_models = sorted(all_models.items(), key=lambda x: x[1], reverse=True)
    names = [m[0] for m in sorted_models]
    scores = [m[1] for m in sorted_models]
    
    # Colors — highlight our model
    colors = ["#e74c3c" if "ours" in n else "#3498db" for n in names]
    
    fig, ax = plt.subplots(figsize=(12, 7))
    bars = ax.barh(range(len(names)), scores, color=colors, edgecolor="black", linewidth=0.5)
    
    # Add value labels
    for i, (bar, score) in enumerate(zip(bars, scores)):
        ax.text(score + 0.5, bar.get_y() + bar.get_height()/2,
                f'{score:.1f}%', va='center', fontsize=10, fontweight='bold')
    
    ax.set_yticks(range(len(names)))
    ax.set_yticklabels(names, fontsize=11)
    ax.set_xlabel("pass@1 (%)", fontsize=12)
    ax.set_title("EvalPlus HumanEval+ Benchmark\n(greedy decoding, pass@1)", fontsize=14, fontweight="bold")
    ax.set_xlim(0, 100)
    ax.invert_yaxis()
    ax.grid(axis="x", alpha=0.3)
    
    # Legend
    from matplotlib.patches import Patch
    legend_elements = [
        Patch(facecolor="#e74c3c", label="Our model"),
        Patch(facecolor="#3498db", label="Published scores (EvalPlus leaderboard)"),
    ]
    ax.legend(handles=legend_elements, loc="lower right", fontsize=10)
    
    # Subtitle
    fig.text(0.5, 0.01, "HumanEval+ (164 problems) | Greedy decoding | vLLM backend | L40S 48GB\n"
             "Published scores from evalplus.github.io/leaderboard.html",
             ha="center", fontsize=9, color="gray")
    
    plt.tight_layout()
    plt.savefig(CHART_PATH, dpi=150, bbox_inches="tight")
    print(f"Chart saved to {CHART_PATH}")


def main():
    if not Path(OUR_MODEL).exists():
        print(f"ERROR: Merged model not found at {OUR_MODEL}")
        sys.exit(1)
    
    print("Running EvalPlus HumanEval+ on our model only...")
    print("Comparison scores will come from the public EvalPlus leaderboard.\n")
    
    result = run_evalplus(OUR_MODEL)
    
    our_score = result.get("plus_pass_at_1") or result.get("pass_at_1") or 0.0
    
    # Save results
    output = {
        "our_model": {
            "path": OUR_MODEL,
            "pass_at_1": result.get("pass_at_1"),
            "plus_pass_at_1": result.get("plus_pass_at_1"),
            "base_pass_at_1": result.get("base_pass_at_1"),
            "elapsed_s": result["elapsed_s"],
            "returncode": result["returncode"],
        },
        "published_scores": PUBLISHED_SCORES,
        "stdout": result["stdout"][-5000:],
    }
    
    with open(JSON_PATH, "w") as f:
        json.dump(output, f, indent=2)
    
    print(f"\n{'='*60}")
    print(f"RESULT: Our model HumanEval+ pass@1 = {our_score:.1f}%")
    print(f"Elapsed: {result['elapsed_s']:.0f}s ({result['elapsed_s']/60:.1f} min)")
    print(f"{'='*60}\n")
    
    # Print comparison table
    print(f"{'Model':<40} {'HumanEval+ pass@1':>20}")
    print("-" * 62)
    print(f"{'Adaptive Operator v4.1 (ours)':<40} {our_score:>19.1f}%")
    for name, score in sorted(PUBLISHED_SCORES.items(), key=lambda x: x[1], reverse=True):
        marker = " <" if score < our_score else (" >" if score > our_score else " =")
        print(f"{name:<40} {score:>19.1f}%{marker}")
    
    # Generate chart
    generate_chart(our_score)
    
    print(f"\nResults saved to {JSON_PATH}")
    print(f"Chart saved to {CHART_PATH}")


if __name__ == "__main__":
    main()