File size: 1,699 Bytes
f5628ad 39cba11 f5628ad 39cba11 f5628ad 39cba11 f5628ad 39cba11 f5628ad 39cba11 f5628ad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from server.eval.precision import run_batch_precision_eval_multi_k
from server.utils import load_config
def main():
config = load_config()
default_path = config.get("eval", {}).get("ground_truth_path", "data/ground_truth/eval_pairs.json")
eval_config = config.get("eval", {})
default_ks = eval_config.get("precision_ks")
if default_ks is None:
default_ks = [eval_config.get("precision_k", 5)]
parser = argparse.ArgumentParser(description="Run batch Precision@K evaluation")
parser.add_argument("--queries", type=str, default=default_path, help="Path to eval_pairs.json")
parser.add_argument("--ks", type=int, nargs="+", default=default_ks, help="K values for Precision@K")
args = parser.parse_args()
print(f"Running Precision eval for K={args.ks} on queries from {args.queries}...")
results = run_batch_precision_eval_multi_k(args.queries, ks=args.ks)
for k in args.ks:
metric_key = f"precision@{k}"
metric_results = results[metric_key]
print(f"\nMean Precision@{k}: {metric_results['mean_precision_at_k']}")
print("\nPer-query breakdown:")
for r in metric_results["per_query_results"]:
print(f" P@{k}={r['precision_at_k']:.2f} | {r['query'][:70]}")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_path = f"eval_results_{timestamp}.json"
with open(output_path, "w") as f:
json.dump(results, f, indent=2)
print(f"\nResults saved to: {output_path}")
if __name__ == "__main__":
main()
|