File size: 3,561 Bytes
6f2de72 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """
ProofyX benchmark runner.
Wraps training/evaluate.py and outputs structured JSON results with
a markdown summary table.
Usage:
python scripts/run_benchmarks.py
python scripts/run_benchmarks.py --samples 5000
python scripts/run_benchmarks.py --output evaluation/results/custom.json
"""
import sys
import os
import json
import argparse
from datetime import datetime, timezone
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if ROOT_DIR not in sys.path:
sys.path.insert(0, ROOT_DIR)
os.environ.setdefault("HF_HOME", os.path.join(ROOT_DIR, ".hf_cache"))
os.environ.setdefault("HF_DATASETS_CACHE", os.path.join(ROOT_DIR, ".hf_cache", "datasets"))
import torch
from training.evaluate import evaluate_models, load_portrait_dataset
def generate_markdown_table(results: dict) -> str:
"""Generate a markdown table from evaluation results."""
header = "| Model | Accuracy | Precision | Recall | F1 | AUC-ROC |"
separator = "|-------|----------|-----------|--------|------|---------|"
rows = [header, separator]
for name, m in sorted(results.items(), key=lambda x: -x[1].get("f1", 0)):
row = (
f"| {name} "
f"| {m['accuracy']:.4f} "
f"| {m['precision']:.4f} "
f"| {m['recall']:.4f} "
f"| {m['f1']:.4f} "
f"| {m['auc_roc']:.4f} |"
)
rows.append(row)
return "\n".join(rows)
def main():
parser = argparse.ArgumentParser(description="ProofyX Benchmark Runner")
parser.add_argument(
"--samples", type=int, default=2000,
help="Number of evaluation samples (default: 2000)",
)
parser.add_argument(
"--output", type=str, default=None,
help="Output JSON path (default: evaluation/results/benchmark_<date>.json)",
)
parser.add_argument(
"--skip-per-class", type=int, default=5000,
help="Skip training samples to avoid data leakage (default: 5000)",
)
args = parser.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Device: {device}")
# Load evaluation dataset
print(f"\nLoading evaluation dataset ({args.samples} samples)...")
eval_data, _ = load_portrait_dataset(
max_samples=args.samples,
train_split=1.0,
face_align=False,
skip_per_class=args.skip_per_class,
seed=999,
)
print(f"Evaluation set: {len(eval_data)} samples")
# Run evaluation
results = evaluate_models(eval_data, device)
if not results:
print("No models available for evaluation.")
return
# Build output
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
output = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"device": str(device),
"samples": len(eval_data),
"models": results,
"markdown_table": generate_markdown_table(results),
}
# Determine output path
if args.output:
output_path = os.path.join(ROOT_DIR, args.output)
else:
results_dir = os.path.join(ROOT_DIR, "evaluation", "results")
os.makedirs(results_dir, exist_ok=True)
output_path = os.path.join(results_dir, f"benchmark_{timestamp}.json")
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output, f, indent=2)
print(f"\nResults saved to: {output_path}")
print(f"\n{output['markdown_table']}")
if __name__ == "__main__":
main()
|