releaseops_arena/eval_api.py

import subprocess
import os
import json
from pathlib import Path

from fastapi import APIRouter, BackgroundTasks

from releaseops_arena.space_paths import get_outputs_root

router = APIRouter()


def _eval_result_path() -> Path:
    return get_outputs_root() / "eval_api_result.json"

@router.post("/run-eval")
def run_eval(background_tasks: BackgroundTasks, limit: int = 3, model_id: str = "hiitsesh/releaseops-grpo-1.7b-best", subfolder: str = "best_by_loss"):
    """
    Triggers an evaluation run. Check HF space logs (stdout) for progress.
    """
    def run_script(limit_val, model_val, subfolder_val):
        print(f"=== Starting Evaluation for {model_val} (subfolder: {subfolder_val}) with limit {limit_val} ===", flush=True)
        out_json = str(_eval_result_path())
        cmd = [
            "python", "training/evaluate_llm_baseline.py",
            "--backend", "torch",
            "--torch-model", model_val,
            "--limit", str(limit_val),
            "--output-json", out_json,
        ]
        
        if subfolder_val:
            cmd.extend(["--torch-subfolder", subfolder_val])
            
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            print("=== Evaluation completed ===", flush=True)
            print("STDOUT:", result.stdout, flush=True)
        except subprocess.CalledProcessError as e:
            print("=== Evaluation failed ===", flush=True)
            print("STDERR:", e.stderr, flush=True)
            
    background_tasks.add_task(run_script, limit, model_id, subfolder)
    return {"message": f"Evaluation started for {model_id} (subfolder={subfolder}, limit={limit}). Check logs."}

@router.get("/get-eval-results")
def get_eval_results():
    path = _eval_result_path()
    if not path.is_file():
        return {"status": "pending_or_missing", "message": "Result file not found yet."}
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)