from __future__ import annotations import difflib from typing import Any, Mapping def compute_agreement(baseline_text: str, candidate_text: str) -> float: if not baseline_text and not candidate_text: return 1.0 return round(difflib.SequenceMatcher(a=baseline_text, b=candidate_text).ratio(), 4) def compute_speedup(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> float: baseline_latency = float(baseline.get("latency_ms_per_token") or 0.0) candidate_latency = float(candidate.get("latency_ms_per_token") or 0.0) if baseline_latency <= 0.0 or candidate_latency <= 0.0: return 1.0 return round(baseline_latency / candidate_latency, 4) def compute_memory_reduction(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> float: baseline_bytes = float(baseline.get("kv_bytes") or 0.0) candidate_bytes = float(candidate.get("kv_bytes") or 0.0) if baseline_bytes <= 0.0 or candidate_bytes <= 0.0: return 1.0 return round(baseline_bytes / candidate_bytes, 4) def build_summary_sentence(agreement: float, speedup: float, memory_reduction: float) -> str: if agreement >= 0.995: quality = "Same answer" elif agreement >= 0.97: quality = "Near-identical answer" elif agreement >= 0.9: quality = "Mostly aligned answer" else: quality = "Visible output drift" return f"{quality}, {speedup:.2f}x faster, {memory_reduction:.2f}x smaller KV footprint."