engines/compare.py

from __future__ import annotations

import difflib
from typing import Any, Mapping


def compute_agreement(baseline_text: str, candidate_text: str) -> float:
    if not baseline_text and not candidate_text:
        return 1.0
    return round(difflib.SequenceMatcher(a=baseline_text, b=candidate_text).ratio(), 4)


def compute_speedup(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> float:
    baseline_latency = float(baseline.get("latency_ms_per_token") or 0.0)
    candidate_latency = float(candidate.get("latency_ms_per_token") or 0.0)
    if baseline_latency <= 0.0 or candidate_latency <= 0.0:
        return 1.0
    return round(baseline_latency / candidate_latency, 4)


def compute_memory_reduction(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> float:
    baseline_bytes = float(baseline.get("kv_bytes") or 0.0)
    candidate_bytes = float(candidate.get("kv_bytes") or 0.0)
    if baseline_bytes <= 0.0 or candidate_bytes <= 0.0:
        return 1.0
    return round(baseline_bytes / candidate_bytes, 4)


def build_summary_sentence(agreement: float, speedup: float, memory_reduction: float) -> str:
    if agreement >= 0.995:
        quality = "Same answer"
    elif agreement >= 0.97:
        quality = "Near-identical answer"
    elif agreement >= 0.9:
        quality = "Mostly aligned answer"
    else:
        quality = "Visible output drift"
    return f"{quality}, {speedup:.2f}x faster, {memory_reduction:.2f}x smaller KV footprint."