solar_eval/evaluators/lcs_diff.py

from typing import Any

from solar_eval.evaluators.base import BaseEvaluator
from solar_eval.evaluators.scoring import compute_proofread_metrics
from solar_eval.models.sample import EvalSample
from solar_eval.providers.base import BaseProvider


class LCSDiffEvaluator(BaseEvaluator):
    """Token-level LCS diff evaluator for proofreading.

    Computes precision and recall by comparing golden corrections vs predicted corrections.
    3-way 비교다 -- "정답이 원문에서 무엇을 고쳤나" 대 "모델이 원문에서 무엇을 고쳤나"를
    비교하는 구조라 원문(`sample.input`)이 없으면 성립하지 않는다
    (마이그레이션 계획 §3).
    """

    required_fields = frozenset({"input", "output", "reference"})

    def __init__(self, metrics: list[str] | None = None) -> None:
        self.metrics = metrics or ["precision", "recall", "tp", "fp", "missing", "redundant"]

    async def evaluate(
        self,
        sample: EvalSample,
        provider: BaseProvider | None = None,
        judge_model: str = "gpt-4o",
    ) -> dict[str, Any]:
        original = sample.input.get("original", "")
        result = compute_proofread_metrics(original, str(sample.reference), sample.output)
        return {
            "score": result["f1"],  # Use F1 as primary score
            "details": result,
        }

    def aggregate(self, results: list[dict[str, Any]]) -> dict[str, Any]:
        if not results:
            return {"overall_score": 0.0, "scores": {}}

        total_tp = sum(r["details"]["tp"] for r in results)
        total_fp = sum(r["details"]["fp"] for r in results)
        total_missing = sum(r["details"]["missing"] for r in results)
        total_redundant = sum(r["details"]["redundant"] for r in results)

        no_golden = (total_tp + total_fp + total_missing) == 0
        no_pred = (total_tp + total_fp + total_redundant) == 0

        if no_golden and no_pred:
            precision, recall = 100.0, 100.0
        else:
            precision = (
                total_tp / (total_tp + total_fp + total_redundant) * 100 if not no_pred else 0.0
            )
            recall = (
                total_tp / (total_tp + total_fp + total_missing) * 100 if not no_golden else 0.0
            )
        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0

        return {
            "overall_score": f1,
            "scores": {
                "f1": f1,
                "precision": precision,
                "recall": recall,
                "tp": total_tp,
                "fp": total_fp,
                "missing": total_missing,
                "redundant": total_redundant,
            },
            "num_samples": len(results),
        }