Spaces:
Sleeping
Sleeping
File size: 2,263 Bytes
75c7554 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | import os
import sys
import argparse
import json
from dotenv import load_dotenv
# Ensure project root is on sys.path
_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
if _ROOT not in sys.path:
sys.path.insert(0, _ROOT)
# Import evaluation logic
from train.evaluate import evaluate_model, score_model, OpenEnvClient, start_server
from backend.api.llm_evaluator import get_llm_analysis
def main():
parser = argparse.ArgumentParser(description="BESS-RL LLM Evaluation CLI")
parser.add_argument("--task", type=str, default="hard", choices=["easy", "medium", "hard"], help="Task complexity")
parser.add_argument("--seeds", type=int, default=5, help="Number of evaluation seeds")
parser.add_argument("--model", type=str, default=None, help="Specific model name (defaults to best_model_<task>)")
args = parser.parse_args()
# Load API Key
load_dotenv(os.path.join(_ROOT, ".env"))
model_name = args.model if args.model else f"best_model_{args.task}"
model_path = os.path.join(_ROOT, "train", "models", model_name)
# Initialize Client and Server
server_process = start_server()
client = OpenEnvClient(base_url="http://127.0.0.1:8000")
try:
eval_seeds = list(range(300, 300 + args.seeds))
results = evaluate_model(client, model_path, args.task, eval_seeds)
scores, overall = score_model(results, args.task)
# Prepare data for LLM
llm_input = {
**results,
"task": args.task,
"model_name": model_name,
"num_seeds": args.seeds,
"scores": {**scores, "overall": overall}
}
# Call LLM Evaluator
analysis = get_llm_analysis(llm_input)
# OUTPUT: Only Score and Verdict as requested
final_score = analysis.get('score', overall)
verdict = analysis.get('verdict', 'N/A')
# print(f"Overall Score (AI Evaluated): {final_score:.3f}")
provider = analysis.get('provider', 'AI')
print(f"Overall Score ({provider} Evaluated): {final_score:.3f}")
print(f"AI Verdict: {verdict}")
finally:
if server_process:
server_process.terminate()
if __name__ == "__main__":
main()
|