File size: 633 Bytes
13c5606 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | """Metrics shared by source reproductions and basecode runs."""
from __future__ import annotations
from dataclasses import asdict
from typing import Any
from .core import DecodeResult
def exact_match(prediction: str, reference: str) -> float:
return float(prediction.strip() == reference.strip())
def summarize_result(result: DecodeResult, reference: str | None = None) -> dict[str, Any]:
payload = asdict(result)
payload["tokens_per_second"] = len(result.tokens) / max(result.elapsed_s, 1e-12)
if reference is not None:
payload["exact_match"] = exact_match(result.text, reference)
return payload
|