File size: 5,814 Bytes
eca5751 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 | """Benchmark Suite - Đánh giá model trên multiple benchmarks."""
from __future__ import annotations
from typing import Dict, Any, List, Optional, Callable
from dataclasses import dataclass, field
from enum import Enum
class BenchmarkType(str, Enum):
MMLU = "mmlu" # General knowledge
HUMANEVAL = "humaneval" # Code generation
GSM8K = "gsm8k" # Math reasoning
BBH = "bbh" # Big-bench hard
truthful_qa = "truthful_qa"
MT_BENCH = "mt_bench" # Multi-turn chat
VI_BENCH = "vi_bench" # Vietnamese specific
@dataclass
class Benchmark:
"""Một benchmark evaluation."""
name: str
type: BenchmarkType
description: str
num_examples: int
languages: List[str] = field(default_factory=lambda: ["en"])
metrics: List[str] = field(default_factory=lambda: ["accuracy"])
estimated_time_minutes: int = 30
class BenchmarkSuite:
"""Run model on multiple benchmarks.
Usage:
suite = BenchmarkSuite()
suite.add(Benchmark(name="humaneval", ...))
results = suite.run(model, tokenizer)
"""
SUPPORTED_BENCHMARKS = [
Benchmark(
name="humaneval",
type=BenchmarkType.HUMANEVAL,
description="HumanEval - Code generation (164 problems)",
num_examples=164,
languages=["en"],
metrics=["pass@1", "pass@10"],
estimated_time_minutes=60,
),
Benchmark(
name="mbpp",
type=BenchmarkType.HUMANEVAL,
description="MBPP - Mostly Basic Python Problems (974 problems)",
num_examples=974,
languages=["en"],
metrics=["pass@1"],
estimated_time_minutes=90,
),
Benchmark(
name="gsm8k",
type=BenchmarkType.GSM8K,
description="Grade School Math 8K",
num_examples=1319,
languages=["en"],
metrics=["accuracy"],
estimated_time_minutes=45,
),
Benchmark(
name="mmlu",
type=BenchmarkType.MMLU,
description="Massive Multitask Language Understanding",
num_examples=14042,
languages=["en"],
metrics=["accuracy"],
estimated_time_minutes=120,
),
Benchmark(
name="bbh",
type=BenchmarkType.BBH,
description="BIG-Bench Hard (23 tasks)",
num_examples=6511,
languages=["en"],
metrics=["accuracy"],
estimated_time_minutes=180,
),
Benchmark(
name="truthful_qa",
type=BenchmarkType.truthful_qa,
description="TruthfulQA - Measure truthfulness",
num_examples=817,
languages=["en"],
metrics=["truthful", "informative"],
estimated_time_minutes=20,
),
Benchmark(
name="mt_bench",
type=BenchmarkType.MT_BENCH,
description="Multi-turn benchmark for chat assistants",
num_examples=80,
languages=["en"],
metrics=["gpt4_score", "judge_score"],
estimated_time_minutes=30,
),
Benchmark(
name="vi_bench",
type=BenchmarkType.VI_BENCH,
description="Vietnamese language understanding",
num_examples=500,
languages=["vi"],
metrics=["accuracy", "fluency"],
estimated_time_minutes=15,
),
]
def __init__(self):
self._benchmarks: Dict[str, Benchmark] = {
b.name: b for b in self.SUPPORTED_BENCHMARKS
}
self._results: Dict[str, Dict] = {}
def add(self, benchmark: Benchmark) -> None:
self._benchmarks[benchmark.name] = benchmark
def list_available(self) -> List[Benchmark]:
return list(self._benchmarks.values())
def run(
self,
model,
tokenizer,
benchmarks: Optional[List[str]] = None,
sample_size: Optional[int] = None,
) -> Dict[str, Dict[str, Any]]:
"""Run benchmarks on model.
Args:
model: NexusCoderForCausalLM
tokenizer: NexusTokenizer
benchmarks: List of benchmark names (None = all)
sample_size: Limit examples per benchmark (for quick eval)
"""
to_run = benchmarks or list(self._benchmarks.keys())
results = {}
for name in to_run:
if name not in self._benchmarks:
results[name] = {"error": f"Unknown benchmark: {name}"}
continue
bench = self._benchmarks[name]
results[name] = {
"status": "not_implemented",
"benchmark": bench.name,
"description": bench.description,
"num_examples": bench.num_examples,
"sample_size": sample_size,
"note": "Evaluation requires downloading dataset. Run scripts/evaluate.py with --download flag.",
}
self._results = results
return results
def summary(self) -> str:
"""Generate summary report."""
if not self._results:
return "No results yet. Run benchmarks first."
lines = ["Benchmark Results Summary", "=" * 50]
for name, result in self._results.items():
if "error" in result:
lines.append(f" {name}: ERROR - {result['error']}")
elif "scores" in result:
lines.append(f" {name}: {result['scores']}")
else:
lines.append(f" {name}: {result.get('status', 'unknown')}")
return "\n".join(lines)
|