File size: 2,479 Bytes
517cbd2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | #!/usr/bin/env python3
"""
Evaluate a best_program.py file using the eplb evaluator.
Runs multiple times and averages the results.
"""
import sys
import json
from pathlib import Path
from evaluator import evaluate
def main():
if len(sys.argv) < 2:
print("Usage: evaluate_best_program.py <path_to_best_program.py> [num_runs]")
sys.exit(1)
program_path = Path(sys.argv[1])
if not program_path.exists():
print(f"Error: File not found: {program_path}")
sys.exit(1)
num_runs = int(sys.argv[2]) if len(sys.argv) > 2 else 3
print(f"Evaluating: {program_path}")
print(f"Running {num_runs} times and averaging results...")
print("=" * 60)
results = []
for run in range(1, num_runs + 1):
print(f"\n--- Run {run}/{num_runs} ---")
result = evaluate(str(program_path))
if "error" in result:
print(f"❌ Error in run {run}: {result['error']}")
sys.exit(1)
results.append(result)
print(f"Run {run} - Combined Score: {result.get('combined_score', 0.0):.6f}")
# Compute averages
avg_result = {
"balancedness_score_gpu": sum(r.get("balancedness_score_gpu", 0.0) for r in results) / len(results),
"balancedness_score_expert": sum(r.get("balancedness_score_expert", 0.0) for r in results) / len(results),
"times_algorithm": sum(r.get("times_algorithm", 0.0) for r in results) / len(results),
"times_inference": sum(r.get("times_inference", 0.0) for r in results) / len(results),
"speed_score": sum(r.get("speed_score", 0.0) for r in results) / len(results),
"combined_score": sum(r.get("combined_score", 0.0) for r in results) / len(results),
}
print("\n" + "=" * 60)
print("AVERAGED RESULTS (over {} runs):".format(num_runs))
print("=" * 60)
print(json.dumps(avg_result, indent=2))
print("\n" + "-" * 60)
print("Summary:")
print(f"✅ Combined Score: {avg_result['combined_score']:.6f}")
print(f" Balancedness (GPU): {avg_result['balancedness_score_gpu']:.6f}")
print(f" Balancedness (Expert): {avg_result['balancedness_score_expert']:.6f}")
print(f" Speed Score: {avg_result['speed_score']:.6f}")
print(f" Avg Algorithm Time: {avg_result['times_algorithm']:.6f}s")
print(f" Avg Inference Time: {avg_result['times_inference']:.6f}s")
print("-" * 60)
if __name__ == "__main__":
main()
|