sft/evaluate.py

# Automated EvalPlus runner for HumanEval and MBPP benchmarks.
# Using the vLLM backend in greedy mode.

import os
import sys
import subprocess
import time
import json
import re
from pathlib import Path
from datetime import datetime
from huggingface_hub import snapshot_download

MODELS = [
    {
        "name": "Quintus-1.7B",
        "id": "iamrahulreddy/Quintus",
        "is_local": False
    },
    {
        "name": "Qwen3-1.7B-Instruct",
        "id": "Qwen/Qwen3-1.7B",
        "is_local": False
    },
    {
        "name": "Qwen3-1.7B-Base",
        "id": "Qwen/Qwen3-1.7B-Base",
        "is_local": False
    }
]

DATASETS = [
    "humaneval", "mbpp",          # EvalPlus benchmarks
    "gsm8k", "winogrande",        # lm-eval fast benchmarks
    "arc_challenge", "boolq", "piqa"
]
EVALPLUS_DATASETS = {"humaneval", "mbpp"}

LM_EVAL_SHOTS = {
    "gsm8k": "10",
    "winogrande": "5",
    "arc_challenge": "25",
    "boolq": "0",
    "piqa": "0"
}

HF_TOKEN = os.environ.get("HF_TOKEN")
TRUST_REMOTE_CODE = os.environ.get("QUINTUS_TRUST_REMOTE_CODE", "").strip().lower() in {"1", "true", "yes", "on"}

def extract_lm_eval_score(results_dir: Path, task: str) -> str:
    """Finds and extracts the primary score from JSON files outputted by lm-evaluation-harness."""
    for json_path in sorted(results_dir.rglob("*.json"), reverse=True):
        try:
            with open(json_path, encoding="utf-8") as fh:
                data = json.load(fh)
            task_results = data.get("results", {})
            for candidate in (task, f"leaderboard_{task}"):
                if candidate in task_results:
                    task_data = task_results[candidate]
                    # Try common metric names
                    for metric in ["acc,none", "acc_norm,none", "exact_match,strict-match", "exact_match,none"]:
                        if metric in task_data:
                            return f"{task_data[metric]*100:.1f}"
        except Exception:
            continue
    return "N/A"

def is_noise(line: str) -> bool:
    l = line.strip()
    if not l:
        return False
    # Progress bar indicators & block characters
    if any(c in l for c in ["█", "━", "╸", "•", "━━━━━━━━"]):
        return True
    # vLLM, ray, flash_attn, huggingface setup/warnings logs
    noise_keywords = [
        "INFO ", "WARNING ", "DEBUG ", "ERROR ", "(EngineCore",
        "Loading safetensors", "Capturing CUDA graphs",
        "Codegen:", "Downloading dataset", "downloading dataset",
        "Initializing a decoder", "Unknown vLLM environment",
        "world_size=", "Using V2 Model Runner", "Model loading took",
        "Using FLASH_ATTN", "Using FlashAttention", "Kernel JIT monitor",
        "autotuner.py", "autotuning", "Autotuning", "loading weights",
        "Loading weights", "Failed to get device capability", "Sanitized code outputs",
        "Raw outputs will be saved", "init engine", "Dynamo bytecode",
        "Directly load the compiled graph", "Directly load AOT compilation", "torch.compile took"
    ]
    if any(k.lower() in l.lower() for k in noise_keywords):
        return True
    # TQDM lines (e.g. 100%|... [00:17<00:00, 9.45it/s])
    if "%|" in l and ("it/s" in l or "s/it" in l):
        return True
    return False

def main():
    print("=" * 80)
    print("              EVALPLUS BENCHMARK RUNNER (HUMANEVAL & MBPP)")
    print("=" * 80)
    print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"Models to evaluate: {[m['name'] for m in MODELS]}")
    print(f"Datasets: {DATASETS}")
    print("=" * 80)

    # Set optional HF token and runtime configuration.
    if HF_TOKEN:
        os.environ["HF_TOKEN"] = HF_TOKEN
    os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
    os.environ["VLLM_MAX_MODEL_LEN"] = "4096"

    # Step 1: Pre-download and prepare model caches
    print("\n--- STAGE 1: WARMING UP MODEL WEIGHTS CACHE ---")

    # Cache all models
    for model in MODELS:
        if model["is_local"]:
            continue
        print(f"\n[DOWNLOADING] Fetching cache for {model['name']} ({model['id']})...")
        try:
            snapshot_download(
                repo_id=model["id"],
                token=HF_TOKEN or None
            )
            print(f"[DOWNLOAD SUCCESS] {model['name']} is cached and ready.")
        except Exception as e:
            print(f"[DOWNLOAD WARNING] Could not pre-download model {model['name']} via snapshot_download: {e}")
            print("The evaluation run will attempt to download it directly during execution.")

    print("\n--- STAGE 2: SEQUENTIAL EVALPLUS EVALUATION ---")
    results = []

    # Run evaluations sequentially
    for model in MODELS:
        # Resolve path
        model_path = str(Path(model["id"]).resolve()) if model["is_local"] else model["id"]

        for dataset in DATASETS:
            print(f"\n[STARTING] Evaluating {model['name']} on {dataset}...")
            print("-" * 60)

            if dataset in EVALPLUS_DATASETS:
                cmd = [
                    sys.executable, "-m", "evalplus.evaluate",
                    "--model", model_path,
                    "--dataset", dataset,
                    "--backend", "vllm",
                    "--greedy"
                ]
            else:
                shots = LM_EVAL_SHOTS.get(dataset, "0")
                out_dir = Path("eval_results") / model["name"] / dataset
                out_dir.mkdir(parents=True, exist_ok=True)

                model_args = (
                    f"pretrained={model_path},dtype=bfloat16,"
                    f"trust_remote_code={str(TRUST_REMOTE_CODE).lower()},"
                    "gpu_memory_utilization=0.9,max_model_len=4096"
                )
                cmd = [
                    sys.executable, "-m", "lm_eval",
                    "--model", "vllm",
                    "--model_args", model_args,
                    "--tasks", dataset,
                    "--num_fewshot", shots,
                    "--batch_size", "auto",
                    "--output_path", str(out_dir),
                    "--log_samples"
                ]
                if dataset == "gsm8k":
                    cmd.extend(["--gen_kwargs", "max_gen_toks=512"])

            print(f"Running command: {' '.join(cmd)}")
            start_time = time.time()

            try:
                # Run the command and stream output
                process = subprocess.Popen(
                    cmd,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                    text=True,
                    bufsize=1
                )

                # Stream and capture output (filtering out vLLM and progress bar noise)
                stdout_text = ""
                for line in process.stdout:
                    stdout_text += line
                    if not is_noise(line):
                        print(line, end="")

                process.wait()
                duration = time.time() - start_time
                time.sleep(5)  # Let OS/driver fully reclaim GPU VRAM before starting next subprocess

                score_str = "N/A"
                if process.returncode == 0:
                    print(f"[SUCCESS] Completed {model['name']} on {dataset} in {duration:.1f} seconds.")

                    # Parse scores
                    if dataset in EVALPLUS_DATASETS:
                        # Find all pass@1 scores
                        matches = re.findall(r"pass@1:\s+([0-9.]+)", stdout_text)
                        if len(matches) >= 2:
                            val0 = float(matches[0])
                            val1 = float(matches[1])
                            if val0 <= 1.0: val0 *= 100
                            if val1 <= 1.0: val1 *= 100
                            score_str = f"Base: {val0:.1f} | Plus: {val1:.1f}"
                        elif len(matches) == 1:
                            val0 = float(matches[0])
                            if val0 <= 1.0: val0 *= 100
                            score_str = f"Base: {val0:.1f}"
                    else:
                        score_str = extract_lm_eval_score(out_dir, dataset)

                    results.append({
                        "model": model["name"],
                        "dataset": dataset,
                        "status": "Success",
                        "score": score_str,
                        "duration": f"{duration/60:.1f} min"
                    })
                else:
                    print(f"[ERROR] command failed with exit code {process.returncode}")
                    results.append({
                        "model": model["name"],
                        "dataset": dataset,
                        "status": f"Failed ({process.returncode})",
                        "score": "ERROR",
                        "duration": f"{duration/60:.1f} min"
                    })

            except Exception as e:
                duration = time.time() - start_time
                print(f"[ERROR] Failed to run benchmark: {e}")
                results.append({
                    "model": model["name"],
                    "dataset": dataset,
                    "status": f"Error",
                    "score": "ERROR",
                    "duration": f"{duration/60:.1f} min"
                })
            print("-" * 60)

    # Print and save summary report
    report_lines = []
    report_lines.append("\n" + "=" * 100)
    report_lines.append("                       BENCHMARK RUN SUMMARY")
    report_lines.append("=" * 100)
    report_lines.append(f"| {'Model':<30} | {'Dataset':<15} | {'Score':<25} | {'Status':<10} | {'Time':<8} |")
    report_lines.append(f"|{'-'*32}|{'-'*17}|{'-'*27}|{'-'*12}|{'-'*10}|")
    for r in results:
        report_lines.append(f"| {r['model']:<30} | {r['dataset']:<15} | {r['score']:<25} | {r['status']:<10} | {r['duration']:<8} |")
    report_lines.append("=" * 100)

    report_text = "\n".join(report_lines)
    print(report_text)
    print("\nNote: Results are saved in the default EvalPlus directory and eval_results/.")

    # Save to file
    with open("qwen_quintus_scores.txt", "w", encoding="utf-8") as f:
        f.write(report_text + "\n")
    print("\n[SUCCESS] Final score report saved to 'qwen_quintus_scores.txt'")

if __name__ == "__main__":
    main()