"""판정 초안·생산자 집계 조립 (계획 2026-09-26 L3b §2.4·D2·D4). - 결과 행마다 판정 3개, 순서 = `exact` → `lcs_f1` → `spans`. 실패 행은 세 지표 모두 `status="error"`·같은 `reason` 이다. - `producer_aggregates.json`: `exact`·`lcs_f1` 의 `mean` 은 파생 참조 구현 `derive("mean")`, `spans` 의 `micro_prf@1:{f1,precision, recall}` 은 **레거시 구현** `LCSDiffEvaluator().aggregate()` 에 ok 행 카운트를 넣은 값이다(D4) — hub 적재의 `producer_aggregate_mismatch` 대조가 "SQL 파생 ↔ 레거시 코드" 대조가 된다. ok 행이 0 이면 `spans` 항목을 싣지 않는다 (레거시의 0.0 은 "측정 없음" 과 구별되지 않는다). """ from __future__ import annotations from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any from poc_bundle.scoring.derivations import derive from poc_bundle.scoring.writer import JudgmentDraft from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator from solar_eval.rescore.evaluator_doc import ACTOR from solar_eval.rescore.proofread_diff_v2 import LABEL_CORRECT, RowError, RowScore EXACT, LCS_F1, SPANS = "exact", "lcs_f1", "spans" #: 판정 파일 안 지표 순서. CRITERIA: tuple[str, ...] = (EXACT, LCS_F1, SPANS) PRODUCER_AGGREGATES_SCHEMA = "producer-aggregates/v1" #: 레거시 `aggregate()["scores"]` 키 → `micro_prf@1` 출력 이름. SPAN_OUTPUTS: tuple[str, ...] = ("f1", "precision", "recall") @dataclass(frozen=True) class ScoredRow: """채점한 결과 행 하나 — `sample_idx`, run 번들 `items.jsonl` 의 키 쌍, 채점 결과.""" sample_idx: int input_key: str golden_key: str outcome: RowScore | RowError def judgment_drafts(run_id: str, rows: Sequence[ScoredRow], created_at: str) -> list[JudgmentDraft]: """행마다 `exact`·`lcs_f1`·`spans` 초안 3개(행 순서 그대로).""" drafts: list[JudgmentDraft] = [] for row in rows: common = { "level": "item", "run_id": run_id, "sample_idx": row.sample_idx, "input_key": row.input_key, "golden_key": row.golden_key, "source_kind": "code", "actor": ACTOR, "created_at": created_at, } for criterion in CRITERIA: fields = {**common, "criterion_id": criterion, **_result_fields(criterion, row.outcome)} drafts.append(JudgmentDraft(fields)) return drafts def _result_fields(criterion: str, outcome: RowScore | RowError) -> dict[str, Any]: if isinstance(outcome, RowError): return {"status": "error", "reason": outcome.reason} if criterion == EXACT: return { "score": outcome.score_exact, "label": outcome.label, "passed": outcome.label == LABEL_CORRECT, "status": "ok", } if criterion == LCS_F1: return {"score": outcome.lcs_f1, "status": "ok"} return {"value": dict(outcome.spans), "status": "ok"} def producer_aggregates( rows: Sequence[ScoredRow], drafts: Sequence[JudgmentDraft] ) -> dict[str, Any]: """`producer-aggregates/v1` 문서 — 값은 (metric_key, aggregation) 사전순.""" values = [_mean_value(EXACT, drafts), _mean_value(LCS_F1, drafts)] ok = [row.outcome for row in rows if isinstance(row.outcome, RowScore)] if ok: scores = legacy_span_scores(ok) for output in SPAN_OUTPUTS: values.append(_value(SPANS, f"micro_prf@1:{output}", scores[output])) values.sort(key=lambda v: (v["metric_key"], v["aggregation"])) return {"schema": PRODUCER_AGGREGATES_SCHEMA, "values": values} def legacy_span_scores(ok: Sequence[RowScore]) -> Mapping[str, Any]: """레거시 `LCSDiffEvaluator.aggregate` 가 ok 행 카운트로 내는 run 층 `scores`(퍼센트).""" aggregated = LCSDiffEvaluator().aggregate([{"details": dict(row.spans)} for row in ok]) return aggregated["scores"] def _mean_value(criterion: str, drafts: Sequence[JudgmentDraft]) -> dict[str, Any]: rows = [d.fields for d in drafts if d.fields.get("criterion_id") == criterion] return _value(criterion, "mean", derive("mean", rows)["value"]) def _value(metric_key: str, aggregation: str, value: Any) -> dict[str, Any]: return { "metric_key": metric_key, "aggregation": aggregation, "variant": "raw", "error_policy": "exclude", "value": value, }