File size: 2,754 Bytes
eca5751 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """
Script đánh giá model trên benchmarks
======================================
Usage:
python scripts/evaluate.py --model model.pt --benchmarks humaneval,gsm8k
python scripts/evaluate.py --model model.pt --benchmarks all --sample-size 100
"""
import sys
import os
import argparse
import json
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import torch
from nexus.config import get_config_by_name
from nexus.model.nexus_coder import NexusCoderForCausalLM
from nexus.tokenizer.tokenizer import NexusTokenizer
from nexus.eval.benchmarks import BenchmarkSuite
from nexus.eval.metrics import compute_perplexity
def main():
parser = argparse.ArgumentParser(description="Nexus Coder Evaluator")
parser.add_argument("--model", type=str, required=True, help="Path to model checkpoint")
parser.add_argument("--config", type=str, default="large", help="Model config")
parser.add_argument(
"--benchmarks",
type=str,
default="humaneval",
help="Comma-separated benchmark names",
)
parser.add_argument("--sample-size", type=int, default=None, help="Limit examples per benchmark")
parser.add_argument("--output", type=str, default="./eval_results.json", help="Output file")
args = parser.parse_args()
print("=" * 60)
print(" NEXUS CODER v0.2 - EVALUATION")
print("=" * 60)
# Load model
config = get_config_by_name(args.config)
model = NexusCoderForCausalLM(config)
if os.path.exists(args.model):
checkpoint = torch.load(args.model, map_location="cpu", weights_only=False)
if "model_state_dict" in checkpoint:
model.load_state_dict(checkpoint["model_state_dict"])
else:
model.load_state_dict(checkpoint)
print(f"✓ Loaded model from {args.model}")
else:
print(f"⚠️ Model file not found, using random init: {args.model}")
# Tokenizer
tokenizer = NexusTokenizer()
# Benchmarks
benchmarks = args.benchmarks.split(",") if args.benchmarks != "all" else None
suite = BenchmarkSuite()
print(f"\n📋 Available benchmarks: {len(suite.list_available())}")
for b in suite.list_available():
print(f" - {b.name}: {b.description}")
print(f"\n🏃 Running benchmarks: {benchmarks or 'all'}")
results = suite.run(model, tokenizer, benchmarks=benchmarks, sample_size=args.sample_size)
# Save results
with open(args.output, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False, default=str)
print(f"\n📊 Results:")
print(suite.summary())
print(f"\n💾 Saved to: {args.output}")
if __name__ == "__main__":
main()
|